From 5690fd0d3304f378754b23b098bd7cb5f4aa1976 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 14:38:21 +0200 Subject: [PATCH 0001/1790] initial commit with latest version extracted from git-python --- .gitignore | 1 + __init__.py | 6 + db.py | 341 +++++++++++++++++++++++++++++++++ fun.py | 115 ++++++++++++ stream.py | 445 ++++++++++++++++++++++++++++++++++++++++++++ test/__init__.py | 1 + test/lib.py | 60 ++++++ test/test_db.py | 90 +++++++++ test/test_stream.py | 172 +++++++++++++++++ test/test_utils.py | 15 ++ utils.py | 38 ++++ 11 files changed, 1284 insertions(+) create mode 100644 .gitignore create mode 100644 __init__.py create mode 100644 db.py create mode 100644 fun.py create mode 100644 stream.py create mode 100644 test/__init__.py create mode 100644 test/lib.py create mode 100644 test/test_db.py create mode 100644 test/test_stream.py create mode 100644 test/test_utils.py create mode 100644 utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..0d20b6487 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/__init__.py b/__init__.py new file mode 100644 index 000000000..5789d7eb7 --- /dev/null +++ b/__init__.py @@ -0,0 +1,6 @@ +"""Initialize the object database module""" + +# default imports +from db import * +from stream import * + diff --git a/db.py b/db.py new file mode 100644 index 000000000..5d3cc6a3f --- /dev/null +++ b/db.py @@ -0,0 +1,341 @@ +"""Contains implementations of database retrieveing objects""" +from git.utils import IndexFileSHA1Writer +from git.errors import ( + InvalidDBRoot, + BadObject, + BadObjectType + ) + +from stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + Sha1Writer, + OStream, + OInfo + ) + +from utils import ( + ENOENT, + to_hex_sha, + exists, + hex_to_bin, + isdir, + mkdir, + rename, + dirname, + join + ) + +from fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) + +import tempfile +import mmap +import os + + +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', + 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) + +class ObjectDBR(object): + """Defines an interface for object database lookup. + Objects are identified either by hex-sha (40 bytes) or + by sha (20 bytes)""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + :return: True if the object identified by the given 40 byte hexsha or 20 bytes + binary sha is contained in the database + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + """ :return: OInfo instance + :param sha: 40 bytes hexsha or 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info_async(self, input_channel): + """Retrieve information of a multitude of objects asynchronously + :param input_channel: Channel yielding the sha's of the objects of interest + :return: Channel yielding OInfo|InvalidOInfo, in any order""" + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + """:return: OStream instance + :param sha: 40 bytes hexsha or 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream_async(self, input_channel): + """Retrieve the OStream of multiple objects + :param input_channel: see ``info`` + :param max_threads: see ``ObjectDBW.store`` + :return: Channel yielding OStream|InvalidOStream instances in any order""" + raise NotImplementedError("To be implemented in subclass") + + #} END query interface + +class ObjectDBW(object): + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """Adjusts the stream to which all data should be sent when storing new objects + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """:return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + def store_async(self, input_channel): + """Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as + they are computed. + + :return: Channel yielding your IStream which served as input, in any order. + The IStreams sha will be set to the sha it received during the process, + or its error attribute will be set to the exception informing about the error. + :param input_channel: Channel yielding IStream instance. + As the same instances will be used in the output channel, you can create a map + between the id(istream) -> istream + :note:As some ODB implementations implement this operation as atomic, they might + abort the whole operation if one item could not be processed. Hence check how + many items have actually been produced.""" + raise NotImplementedError("To be implemented in subclass") + + #} END edit interface + + +class FileDBBase(object): + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + :note: The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super(FileDBBase, self).__init__() + self._root_path = root_path + + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, rela_path) + #} END interface + + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(to_hex_sha(sha))) + try: + fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + raise BadObject(to_hex_sha(sha)) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(to_hex_sha(sha)) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(to_hex_sha(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + writer = FDCompressedSha1Writer(fd) + # END handle custom writer + + try: + try: + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + + sha = istream.sha or writer.sha(as_hex=True) + + if tmp_path: + obj_path = self.db_path(self.object_path(sha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + rename(tmp_path, obj_path) + # END handle dry_run + + istream.sha = sha + return istream + + +class PackedDB(FileDBBase, ObjectDBR): + """A database operating on a set of object packs""" + + +class CompoundDB(ObjectDBR): + """A database which delegates calls to sub-databases""" + + +class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" + + +#class GitObjectDB(CompoundDB, ObjectDBW): +class GitObjectDB(LooseObjectDB): + """A database representing the default git object store, which includes loose + objects, pack files and an alternates file + + It will create objects only in the loose object database. + :note: for now, we use the git command to do all the lookup, just until he + have packs and the other implementations + """ + def __init__(self, root_path, git): + """Initialize this instance with the root and a git command""" + super(GitObjectDB, self).__init__(root_path) + self._git = git + + def info(self, sha): + t = self._git.get_object_header(sha) + return OInfo(*t) + + def stream(self, sha): + """For now, all lookup is done by git itself""" + t = self._git.stream_object_data(sha) + return OStream(*t) + diff --git a/fun.py b/fun.py new file mode 100644 index 000000000..3321a8ea4 --- /dev/null +++ b/fun.py @@ -0,0 +1,115 @@ +"""Contains basic c-functions which usually contain performance critical code +Keeping this code separate from the beginning makes it easier to out-source +it into c later, if required""" + +from git.errors import ( + BadObjectType + ) + +import zlib +decompressobj = zlib.decompressobj + + +# INVARIANTS +type_id_to_type_map = { + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag" + } + +# used when dealing with larger streams +chunk_size = 1000*1000 + +__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', + 'write_object' ) + +#{ Routines + +def is_loose_object(m): + """:return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 + +def loose_object_header_info(m): + """:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find("\0")].split(" ") + return type_name, int(size) + +def object_header_info(m): + """:return: tuple(type_string, uncompressed_size_in_bytes + :param mmap: mapped memory map. It will be + seeked to the actual start of the object contents, which can be used + to initialize a zlib decompress object. + :note: This routine can only handle new-style objects which are assumably contained + in packs + """ + assert not is_loose_object(m), "Use loose_object_header_info instead" + + c = b0 # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = ord(m[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + + # finally seek the map to the start of the data stream + m.seek(i) + try: + return (type_id_to_type_map[type_id], size) + except KeyError: + # invalid object type - we could try to be smart now and decode part + # of the stream to get the info, problem is that we had trouble finding + # the exact start of the content stream + raise BadObjectType(type_id) + # END handle exceptions + +def write_object(type, size, read, write, chunk_size=chunk_size): + """Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write("%s %i\0" % (type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw + +def stream_copy(read, write, size, chunk_size): + """Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + :note: its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size-dbw) + data_len = write(read(cs)) + dbw += data_len + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + + +#} END routines diff --git a/stream.py b/stream.py new file mode 100644 index 000000000..da97cf5b6 --- /dev/null +++ b/stream.py @@ -0,0 +1,445 @@ +import zlib +from cStringIO import StringIO +from git.utils import make_sha +import errno + +from utils import ( + to_hex_sha, + to_bin_sha, + write, + close + ) + +__all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', + 'DecompressMemMapReader', 'FDCompressedSha1Writer') + + +# ZLIB configuration +# used when compressing objects - 1 to 9 ( slowest ) +Z_BEST_SPEED = 1 + + +#{ ODB Bases + +class OInfo(tuple): + """Carries information about an object in an ODB, provdiing information + about the sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.sha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def sha(self): + return self[0] + + @property + def type(self): + return self[1] + + @property + def size(self): + return self[2] + #} END interface + + +class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + #} END stream reader interface + + +class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_sha(self, sha): + self[0] = sha + + def _sha(self): + return self[0] + + sha = property(_sha, _set_sha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def sha(self): + return self[0] + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases + + +#{ RO Streams + +class DecompressMemMapReader(object): + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + :note: The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, thats all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close') + + max_read_size = 512*1024 # currently unused + + def __init__(self, m, close_on_deletion, size): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._close = close_on_deletion # close the memmap on deletion ? + + def __del__(self): + if self._close: + self._m.close() + # END handle resource freeing + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + :return: parsed type_string, size""" + # read header + maxb = 512 # should really be enough, cgit uses 8192 I believe + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find("\0") + type, size = hdr[:hdrend].split(" ") + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 # count terminating \0 + self._buf = StringIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + return type, size + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + type, size = inst._parse_header_info() + return type, size, inst + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return str() + # END handle depletion + + # protect from memory peaks + # If he tries to read large chunks, our memory patterns get really bad + # as we end up copying a possibly huge chunk from our memory map right into + # memory. This might not even be possible. Nonetheless, try to dampen the + # effect a bit by reading in chunks, returning a huge string in the end. + # Our performance now depends on StringIO. This way we don't need two large + # buffers in peak times, but only one large one in the end which is + # the return buffer + # NO: We don't do it - if the user thinks its best, he is right. If he + # has trouble, he will start reading in chunks. According to our tests + # its still faster if we read 10 Mb at once instead of chunking it. + + # if size > self.max_read_size: + # sio = StringIO() + # while size: + # read_size = min(self.max_read_size, size) + # data = self.read(read_size) + # sio.write(data) + # size -= len(data) + # if len(data) < read_size: + # break + # # END data loop + # sio.seek(0) + # return sio.getvalue() + # # END handle maxread + # + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = str() + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes durin g + # the next read. + tail = self._zip.unconsumed_tail + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + size + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + size + # END handle tail + + + # if window is too small, make it larger so zip can decompress something + win_size = self._cwe - self._cws + if win_size < 8: + self._cwe = self._cws + 8 + # END adjust winsize + indata = self._m[self._cws:self._cwe] # another copy ... :( + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + + dcompdat = self._zip.decompress(indata, size) + + self._br += len(dcompdat) + if dat: + dcompdat = dat + dcompdat + + return dcompdat + +#} END RO streams + + +#{ W Streams + +class Sha1Writer(object): + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha("") + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex = False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface + +class FDCompressedSha1Writer(Sha1Writer): + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + :note: operates on raw file descriptors + :note: for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super(FDCompressedSha1Writer, self).__init__() + self.fd = fd + self.zip = zlib.compressobj(Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): + raise self.exc + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface + +#} END W streams diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test/lib.py b/test/lib.py new file mode 100644 index 000000000..d51997488 --- /dev/null +++ b/test/lib.py @@ -0,0 +1,60 @@ +"""Utilities used in ODB testing""" +from git.odb import ( + OStream, + ) +from git.odb.stream import Sha1Writer + +import zlib +from cStringIO import StringIO + +#{ Stream Utilities + +class DummyStream(object): + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read + + +class DeriveTest(OStream): + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg + + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(1) # fastest + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + +#} END stream utilitiess + diff --git a/test/test_db.py b/test/test_db.py new file mode 100644 index 000000000..35ba86802 --- /dev/null +++ b/test/test_db.py @@ -0,0 +1,90 @@ +"""Test for object db""" +from test.testlib import * +from lib import ZippedStoreShaWriter + +from git.odb import * +from git.odb.stream import Sha1Writer +from git import Blob +from git.errors import BadObject + + +from cStringIO import StringIO +import os + +class TestDB(TestBase): + """Test the different db class implementations""" + + # data + two_lines = "1234\nhello world" + + all_data = (two_lines, ) + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + :note: requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + + istream = IStream(Blob.type, len(data), StringIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.sha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 40 # for now we require 40 byte shas as default + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert Blob.type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == Blob.type + assert ostream.size == len(data) + else: + self.failUnlessRaises(BadObject, db.info, sha) + self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.sha is not None + prev_sha = istream.sha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.sha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode + + @with_bare_rw_repo + def test_writing(self, rwrepo): + ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + + # write data + self._assert_object_writing(ldb) + diff --git a/test/test_stream.py b/test/test_stream.py new file mode 100644 index 000000000..020fe6bd3 --- /dev/null +++ b/test/test_stream.py @@ -0,0 +1,172 @@ +"""Test for object db""" +from test.testlib import * +from lib import ( + DummyStream, + DeriveTest, + Sha1Writer + ) + +from git.odb import * +from git import Blob +from cStringIO import StringIO +import tempfile +import os +import zlib + + + + +class TestStream(TestBase): + """Test stream classes""" + + data_sizes = (15, 10000, 1000*1024+512) + + def test_streams(self): + # test info + sha = Blob.NULL_HEX_SHA + s = 20 + info = OInfo(sha, Blob.type, s) + assert info.sha == sha + assert info.type == Blob.type + assert info.size == s + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # derive with own args + DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(Blob.type, s, stream) + assert istream.sha == None + istream.sha = sha + assert istream.sha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == Blob.type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) / ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i*ss:(i+1)*ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(Blob.type, cdata)) + type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert type == Blob.type + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + def rewind(r): + r._zip = zlib.decompressobj() + r._br = r._cws = r._cwe = 0 + if with_size: + r._parse_header_info() + # END skip header + # END make rewind func + + self._assert_stream_reader(reader, cdata, rewind) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + #zdi# + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write("hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write("hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + # its closed already + self.failUnlessRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY) + written_data = os.read(fd, os.path.getsize(path)) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 000000000..34572b37e --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,15 @@ +"""Test for object db""" +from test.testlib import * +from git import Blob +from git.odb.utils import ( + to_hex_sha, + to_bin_sha + ) + + +class TestUtils(TestBase): + def test_basics(self): + assert to_hex_sha(Blob.NULL_HEX_SHA) == Blob.NULL_HEX_SHA + assert len(to_bin_sha(Blob.NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(Blob.NULL_HEX_SHA)) == Blob.NULL_HEX_SHA + diff --git a/utils.py b/utils.py new file mode 100644 index 000000000..6863e97b9 --- /dev/null +++ b/utils.py @@ -0,0 +1,38 @@ +import binascii +import os +import errno + +#{ Routines + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +isdir = os.path.isdir +rename = os.rename +dirname = os.path.dirname +join = os.path.join +read = os.read +write = os.write +close = os.close + + +#} END Routines + + From 94c2167bb08c65d4354941e14b1899449efb04f7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 16:14:00 +0200 Subject: [PATCH 0002/1790] Adjusted imports and tests to deal with new folder structure --- db.py | 5 +- exc.py | 14 +++++ fun.py | 2 +- stream.py | 6 +-- test/lib.py | 67 +++++++++++++++++++++++- test/performance/lib.py | 49 ++++++++++++++++++ test/performance/test_db.py | 15 ++++++ test/performance/test_stream.py | 91 +++++++++++++++++++++++++++++++++ test/test_db.py | 28 +++++----- test/test_stream.py | 33 +++++++----- test/test_util.py | 15 ++++++ test/test_utils.py | 15 ------ typ.py | 10 ++++ util.py | 75 +++++++++++++++++++++++++++ utils.py | 38 -------------- 15 files changed, 376 insertions(+), 87 deletions(-) create mode 100644 exc.py create mode 100644 test/performance/lib.py create mode 100644 test/performance/test_db.py create mode 100644 test/performance/test_stream.py create mode 100644 test/test_util.py delete mode 100644 test/test_utils.py create mode 100644 typ.py create mode 100644 util.py delete mode 100644 utils.py diff --git a/db.py b/db.py index 5d3cc6a3f..7ec8a24b3 100644 --- a/db.py +++ b/db.py @@ -1,6 +1,5 @@ """Contains implementations of database retrieveing objects""" -from git.utils import IndexFileSHA1Writer -from git.errors import ( +from exc import ( InvalidDBRoot, BadObject, BadObjectType @@ -14,7 +13,7 @@ OInfo ) -from utils import ( +from util import ( ENOENT, to_hex_sha, exists, diff --git a/exc.py b/exc.py new file mode 100644 index 000000000..3eaf5777d --- /dev/null +++ b/exc.py @@ -0,0 +1,14 @@ +"""Module with common exceptions""" + +class ODBError(Exception): + """All errors thrown by the object database""" + +class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + +class BadObject(ODBError): + """The object with the given SHA does not exist""" + +class BadObjectType(ODBError): + """The object had an unsupported type""" + diff --git a/fun.py b/fun.py index 3321a8ea4..80b0f41b6 100644 --- a/fun.py +++ b/fun.py @@ -2,7 +2,7 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from git.errors import ( +from exc import ( BadObjectType ) diff --git a/stream.py b/stream.py index da97cf5b6..309df28c4 100644 --- a/stream.py +++ b/stream.py @@ -1,11 +1,11 @@ import zlib from cStringIO import StringIO -from git.utils import make_sha import errno -from utils import ( +from util import ( to_hex_sha, - to_bin_sha, + to_bin_sha, + make_sha, write, close ) diff --git a/test/lib.py b/test/lib.py index d51997488..071e38a5f 100644 --- a/test/lib.py +++ b/test/lib.py @@ -1,12 +1,75 @@ """Utilities used in ODB testing""" -from git.odb import ( +from gitdb import ( OStream, ) -from git.odb.stream import Sha1Writer +from gitdb.stream import Sha1Writer +import sys import zlib +import random +from array import array from cStringIO import StringIO +import unittest +import tempfile +import shutil +import os + + +#{ Bases + +class TestBase(unittest.TestCase): + """Base class for all tests""" + + +#} END bases + +#{ Decorators + +def with_rw_directory(func): + """Create a temporary directory which can be written to, remove it if the + test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): + path = tempfile.mktemp(suffix=func.__name__) + os.mkdir(path) + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + raise + else: + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + + +#} END decorators + +#{ Routines + +def make_bytes(size_in_bytes, randomize=False): + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tostring() + + +def make_object(type, data): + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata + data + +#} END routines + #{ Stream Utilities class DummyStream(object): diff --git a/test/performance/lib.py b/test/performance/lib.py new file mode 100644 index 000000000..03788c081 --- /dev/null +++ b/test/performance/lib.py @@ -0,0 +1,49 @@ +"""Contains library functions""" +import os +from gitdb.test.lib import * +import shutil +import tempfile + + +#{ Invvariants +k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" +#} END invariants + + +#{ Utilities +def resolve_or_fail(env_var): + """:return: resolved environment variable or raise EnvironmentError""" + try: + return os.environ[env_var] + except KeyError: + raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) + # END exception handling + +#} END utilities + + +#{ Base Classes + +class TestBigRepoR(TestBase): + """TestCase providing access to readonly 'big' repositories using the following + member variables: + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git""" + + #{ Invariants + head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' + head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' + #} END invariants + + @classmethod + def setUpAll(cls): + try: + super(TestBigRepoR, cls).setUpAll() + except AttributeError: + pass + cls.gitrepopath = resolve_or_fail(k_env_git_repo) + + +#} END base classes diff --git a/test/performance/test_db.py b/test/performance/test_db.py new file mode 100644 index 000000000..cd231b650 --- /dev/null +++ b/test/performance/test_db.py @@ -0,0 +1,15 @@ +"""Performance tests for object store""" + +import sys +from time import time + +from lib import ( + TestBigRepoR + ) + +class TestGitDBPerformance(TestBigRepoR): + + def test_random_access(self): + pass + # TODO: use the actual db for this + diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py new file mode 100644 index 000000000..2880c9b79 --- /dev/null +++ b/test/performance/test_stream.py @@ -0,0 +1,91 @@ +"""Performance data streaming performance""" + +from lib import TestBigRepoR +from gitdb.db import * +from gitdb.stream import * + +from cStringIO import StringIO +from time import time +import os +import sys +import stat +import subprocess + + +from lib import ( + TestBigRepoR, + make_bytes, + with_rw_directory + ) + + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) + + +class TestObjDBPerformance(TestBigRepoR): + + large_data_size_bytes = 1000*1000*10 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).sha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(sha) + fsize_kib = os.path.getsize(db_file) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so git has something to do + os.remove(db_file) + + # END for each randomization factor diff --git a/test/test_db.py b/test/test_db.py index 35ba86802..7f58f4f00 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,12 +1,14 @@ """Test for object db""" -from test.testlib import * -from lib import ZippedStoreShaWriter - -from git.odb import * -from git.odb.stream import Sha1Writer -from git import Blob -from git.errors import BadObject +from lib import ( + with_rw_directory, + ZippedStoreShaWriter, + TestBase + ) +from gitdb import * +from gitdb.stream import Sha1Writer +from gitdb.exc import BadObject +from gitdb.typ import str_blob_type from cStringIO import StringIO import os @@ -36,7 +38,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(Blob.type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), StringIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -48,12 +50,12 @@ def _assert_object_writing(self, db): # verify data - the slow way, we want to run code if not dry_run: info = db.info(sha) - assert Blob.type == info.type + assert str_blob_type == info.type assert info.size == len(data) ostream = db.stream(sha) assert ostream.read() == data - assert ostream.type == Blob.type + assert ostream.type == str_blob_type assert ostream.size == len(data) else: self.failUnlessRaises(BadObject, db.info, sha) @@ -81,9 +83,9 @@ def _assert_object_writing(self, db): # END for each data set # END for each dry_run mode - @with_bare_rw_repo - def test_writing(self, rwrepo): - ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + @with_rw_directory + def test_writing(self, path): + ldb = LooseObjectDB(path) # write data self._assert_object_writing(ldb) diff --git a/test/test_stream.py b/test/test_stream.py index 020fe6bd3..af7fdc35a 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -1,13 +1,22 @@ """Test for object db""" -from test.testlib import * from lib import ( + TestBase, DummyStream, DeriveTest, - Sha1Writer + Sha1Writer, + make_bytes, + make_object + ) + +from gitdb import * +from gitdb.util import ( + NULL_HEX_SHA + ) + +from gitdb.typ import ( + str_blob_type ) -from git.odb import * -from git import Blob from cStringIO import StringIO import tempfile import os @@ -23,11 +32,11 @@ class TestStream(TestBase): def test_streams(self): # test info - sha = Blob.NULL_HEX_SHA + sha = NULL_HEX_SHA s = 20 - info = OInfo(sha, Blob.type, s) + info = OInfo(sha, str_blob_type, s) assert info.sha == sha - assert info.type == Blob.type + assert info.type == str_blob_type assert info.size == s # test ostream @@ -40,10 +49,10 @@ def test_streams(self): assert stream.bytes == 20 # derive with own args - DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() # test istream - istream = IStream(Blob.type, s, stream) + istream = IStream(str_blob_type, s, stream) assert istream.sha == None istream.sha = sha assert istream.sha == sha @@ -54,7 +63,7 @@ def test_streams(self): assert istream.size == s istream.size = s * 2 istream.size == s * 2 - assert istream.type == Blob.type + assert istream.type == str_blob_type istream.type = "something" assert istream.type == "something" assert istream.stream is stream @@ -104,10 +113,10 @@ def test_decompress_reader(self): # create reader if with_size: # need object data - zdata = zlib.compress(make_object(Blob.type, cdata)) + zdata = zlib.compress(make_object(str_blob_type, cdata)) type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) - assert type == Blob.type + assert type == str_blob_type else: # here we need content data zdata = zlib.compress(cdata) diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 000000000..5aac5b84b --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,15 @@ +"""Test for object db""" +from lib import TestBase +from gitdb.util import ( + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA + ) + + +class TestUtils(TestBase): + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + diff --git a/test/test_utils.py b/test/test_utils.py deleted file mode 100644 index 34572b37e..000000000 --- a/test/test_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Test for object db""" -from test.testlib import * -from git import Blob -from git.odb.utils import ( - to_hex_sha, - to_bin_sha - ) - - -class TestUtils(TestBase): - def test_basics(self): - assert to_hex_sha(Blob.NULL_HEX_SHA) == Blob.NULL_HEX_SHA - assert len(to_bin_sha(Blob.NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(Blob.NULL_HEX_SHA)) == Blob.NULL_HEX_SHA - diff --git a/typ.py b/typ.py new file mode 100644 index 000000000..54a1f84be --- /dev/null +++ b/typ.py @@ -0,0 +1,10 @@ +"""Module containing information about types known to the database""" + +#{ String types + +str_blob_type = "blob" +str_commit_type = "commit" +str_tree_type = "tree" +str_tag_type = "tag" + +#} END string types diff --git a/util.py b/util.py new file mode 100644 index 000000000..a6f726399 --- /dev/null +++ b/util.py @@ -0,0 +1,75 @@ +import binascii +import os +import errno + +try: + import hashlib +except ImportError: + import sha + + +#{ Aliases + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +isdir = os.path.isdir +rename = os.rename +dirname = os.path.dirname +join = os.path.join +read = os.read +write = os.write +close = os.close + +# constants +NULL_HEX_SHA = "0"*40 + +#} END Aliases + + +#{ Routines + +def make_sha(source=''): + """A python2.4 workaround for the sha/hashlib module fiasco + :note: From the dulwich project """ + try: + return hashlib.sha1(source) + except NameError: + sha1 = sha.sha(source) + return sha1 + +def stream_copy(source, destination, chunk_size=512*1024): + """Copy all data from the source stream into the destination stream in chunks + of size chunk_size + + :return: amount of bytes written""" + br = 0 + while True: + chunk = source.read(chunk_size) + destination.write(chunk) + br += len(chunk) + if len(chunk) < chunk_size: + break + # END reading output stream + return br + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + + +#} END routines + diff --git a/utils.py b/utils.py deleted file mode 100644 index 6863e97b9..000000000 --- a/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -import binascii -import os -import errno - -#{ Routines - -hex_to_bin = binascii.a2b_hex -bin_to_hex = binascii.b2a_hex - -def to_hex_sha(sha): - """:return: hexified version of sha""" - if len(sha) == 40: - return sha - return bin_to_hex(sha) - -def to_bin_sha(sha): - if len(sha) == 20: - return sha - return hex_to_bin(sha) - -# errors -ENOENT = errno.ENOENT - -# os shortcuts -exists = os.path.exists -mkdir = os.mkdir -isdir = os.path.isdir -rename = os.rename -dirname = os.path.dirname -join = os.path.join -read = os.read -write = os.write -close = os.close - - -#} END Routines - - From 93f7316425128a498e0581eca12ef7b44380a1ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 17:18:34 +0200 Subject: [PATCH 0003/1790] Added async as submodule tests: minimal reorganzation of code --- .gitmodules | 3 +++ ext/async | 1 + test/lib.py | 7 ++++++- test/performance/test_stream.py | 9 +-------- 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 .gitmodules create mode 160000 ext/async diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..45ddc0b4c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/async"] + path = ext/async + url = git://gitorious.org/git-python/async.git diff --git a/ext/async b/ext/async new file mode 160000 index 000000000..5a13dc577 --- /dev/null +++ b/ext/async @@ -0,0 +1 @@ +Subproject commit 5a13dc5772ec3b00b75c8e3b533051cfb82c4929 diff --git a/test/lib.py b/test/lib.py index 071e38a5f..f0c4064ab 100644 --- a/test/lib.py +++ b/test/lib.py @@ -62,11 +62,16 @@ def make_bytes(size_in_bytes, randomize=False): a = array('i', producer) return a.tostring() - def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata + data + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) #} END routines diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 2880c9b79..8916d3e58 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -14,18 +14,11 @@ from lib import ( TestBigRepoR, - make_bytes, + make_memory_file, with_rw_directory ) -def make_memory_file(size_in_bytes, randomize=False): - """:return: tuple(size_of_stream, stream) - :param randomize: try to produce a very random stream""" - d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) - - class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it From 10fef8f8e4ee83cf54feadbb5ffb522efec739fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 17:18:51 +0200 Subject: [PATCH 0004/1790] Added project information --- AUTHORS | 1 + README | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 AUTHORS create mode 100644 README diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..490baad8e --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Creator: Sebastian Thiel diff --git a/README b/README new file mode 100644 index 000000000..a52d9f508 --- /dev/null +++ b/README @@ -0,0 +1,41 @@ +GtDB +===== + +GitDB allows you to access bare git repositories for reading and writing. It +aims at allowing full access to loose objects as well as packs with performance +and scalability in mind. It operates exclusively on streams, allowing to operate +on large objects with a small memory footprint. + +REQUIREMENTS +============ + +* Python Nose - for running the tests + +SOURCE +====== +The source is available in a git repository at gitorious and github: + +git://gitorious.org/git-python/gitdb.git +git://github.com/Byron/gitdb.git + +Once the clone is complete, please be sure to initialize the submodules using + + cd gitdb + git submodule update --init + +Run the tests with + + nosetests + +MAILING LIST +============ +http://groups.google.com/group/git-python + +ISSUE TRACKER +============= +http://byronimo.lighthouseapp.com/projects/51787-gitpython + +LICENSE +======= + +New BSD License From c64a9741a648526a3d24780ccd5ca193f48684c5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 20:36:00 +0200 Subject: [PATCH 0005/1790] Implemented all async methods, including test which shows how to chain the async method together --- __init__.py | 12 +++++++ db.py | 55 +++++++++++++++++++++--------- ext/async | 2 +- test/__init__.py | 11 ++++++ test/lib.py | 2 +- test/test_db.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++- util.py | 10 ++++++ 7 files changed, 161 insertions(+), 19 deletions(-) diff --git a/__init__.py b/__init__.py index 5789d7eb7..8b0e47b19 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,17 @@ """Initialize the object database module""" +import sys +import os + +#{ Initialization +def _init_externals(): + """Initialize external projects by putting them into the path""" + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext')) + +#} END initialization + +_init_externals() + # default imports from db import * from stream import * diff --git a/db.py b/db.py index 7ec8a24b3..8107fee25 100644 --- a/db.py +++ b/db.py @@ -14,6 +14,7 @@ ) from util import ( + pool, ENOENT, to_hex_sha, exists, @@ -32,6 +33,11 @@ stream_copy ) + +from async import ( + ChannelThreadTask + ) + import tempfile import mmap import os @@ -40,6 +46,7 @@ __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) + class ObjectDBR(object): """Defines an interface for object database lookup. Objects are identified either by hex-sha (40 bytes) or @@ -52,21 +59,30 @@ def __contains__(self, sha): def has_object(self, sha): """ :return: True if the object identified by the given 40 byte hexsha or 20 bytes - binary sha is contained in the database - :raise BadObject:""" + binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") + def has_object_async(self, reader): + """Return a reader yielding information about the membership of objects + as identified by shas + :param reader: Reader yielding 20 byte or 40 byte shas. + :return: async.Reader yielding tuples of (sha, bool) pairs which indicate + whether the given sha exists in the database or not""" + task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) + return pool.add_task(task) + def info(self, sha): """ :return: OInfo instance :param sha: 40 bytes hexsha or 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def info_async(self, input_channel): + def info_async(self, reader): """Retrieve information of a multitude of objects asynchronously - :param input_channel: Channel yielding the sha's of the objects of interest - :return: Channel yielding OInfo|InvalidOInfo, in any order""" - raise NotImplementedError("To be implemented in subclass") + :param reader: Channel yielding the sha's of the objects of interest + :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" + task = ChannelThreadTask(reader, str(self.info_async), self.info) + return pool.add_task(task) def stream(self, sha): """:return: OStream instance @@ -74,12 +90,17 @@ def stream(self, sha): :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def stream_async(self, input_channel): + def stream_async(self, reader): """Retrieve the OStream of multiple objects - :param input_channel: see ``info`` + :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` - :return: Channel yielding OStream|InvalidOStream instances in any order""" - raise NotImplementedError("To be implemented in subclass") + :return: async.Reader yielding OStream|InvalidOStream instances in any order + :note: depending on the system configuration, it might not be possible to + read all OStreams at once. Instead, read them individually using reader.read(x) + where x is small enough.""" + # base implementation just uses the stream method repeatedly + task = ChannelThreadTask(reader, str(self.stream_async), self.stream) + return pool.add_task(task) #} END query interface @@ -114,7 +135,7 @@ def store(self, istream): :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - def store_async(self, input_channel): + def store_async(self, reader): """Create multiple new objects in the database asynchronously. The method will return right away, returning an output channel which receives the results as they are computed. @@ -122,13 +143,15 @@ def store_async(self, input_channel): :return: Channel yielding your IStream which served as input, in any order. The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. - :param input_channel: Channel yielding IStream instance. - As the same instances will be used in the output channel, you can create a map - between the id(istream) -> istream - :note:As some ODB implementations implement this operation as atomic, they might + :param reader: async.Reader yielding IStream instances. + The same instances will be used in the output channel as were received + in by the Reader. + :note:As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" - raise NotImplementedError("To be implemented in subclass") + # base implementation uses store to perform the work + task = ChannelThreadTask(reader, str(self.store_async), self.store) + return pool.add_task(task) #} END edit interface diff --git a/ext/async b/ext/async index 5a13dc577..164bb702e 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 5a13dc5772ec3b00b75c8e3b533051cfb82c4929 +Subproject commit 164bb702e3871ab30341a714ef517fb58cd76772 diff --git a/test/__init__.py b/test/__init__.py index 8b1378917..0dec7750f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1 +1,12 @@ +import gitdb.util + +#{ Initialization +def _init_pool(): + """Assure the pool is actually threaded""" + size = 2 + print "Setting ThreadPool to %i" % size + gitdb.util.pool.set_size(size) + + +#} END initialization diff --git a/test/lib.py b/test/lib.py index f0c4064ab..fc0982bcb 100644 --- a/test/lib.py +++ b/test/lib.py @@ -30,7 +30,7 @@ def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" def wrapper(self): - path = tempfile.mktemp(suffix=func.__name__) + path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) try: return func(self, path) diff --git a/test/test_db.py b/test/test_db.py index 7f58f4f00..7ba770f48 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -10,6 +10,8 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type +from async import IteratorReader + from cStringIO import StringIO import os @@ -78,10 +80,93 @@ def _assert_object_writing(self, db): new_ostream = db.ostream() # note: only works as long our store write uses the same compression - # level, which is zip + # level, which is zip_best assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode + + def _assert_object_writing_async(self, db): + """Test generic object writing using asynchronous access""" + ni = 5000 + def istream_generator(offset=0, ni=ni): + for data_src in xrange(ni): + data = str(data_src + offset) + yield IStream(str_blob_type, len(data), StringIO(data)) + # END for each item + # END generator utility + + # for now, we are very trusty here as we expect it to work if it worked + # in the single-stream case + + # write objects + reader = IteratorReader(istream_generator()) + istream_reader = db.store_async(reader) + istreams = istream_reader.read() # read all + assert istream_reader.task().error() is None + assert len(istreams) == ni + + for stream in istreams: + assert stream.error is None + assert len(stream.sha) == 40 + assert isinstance(stream, IStream) + # END assert each stream + + # test has-object-async - we must have all previously added ones + reader = IteratorReader( istream.sha for istream in istreams ) + hasobject_reader = db.has_object_async(reader) + count = 0 + for sha, has_object in hasobject_reader: + assert has_object + count += 1 + # END for each sha + assert count == ni + + # read the objects we have just written + reader = IteratorReader( istream.sha for istream in istreams ) + ostream_reader = db.stream_async(reader) + + # read items individually to prevent hitting possible sys-limits + count = 0 + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert ostream_reader.task().error() is None + assert count == ni + + # get info about our items + reader = IteratorReader( istream.sha for istream in istreams ) + info_reader = db.info_async(reader) + + count = 0 + for oinfo in info_reader: + assert isinstance(oinfo, OInfo) + count += 1 + # END for each oinfo instance + assert count == ni + + + # combined read-write using a converter + # add 2500 items, and obtain their output streams + nni = 2500 + reader = IteratorReader(istream_generator(offset=ni, ni=nni)) + istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ] + + istream_reader = db.store_async(reader) + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = db.stream_async(istream_reader) + + count = 0 + # read it individually, otherwise we might run into the ulimit + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert count == nni + + + @with_rw_directory def test_writing(self, path): @@ -89,4 +174,5 @@ def test_writing(self, path): # write data self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) diff --git a/util.py b/util.py index a6f726399..fd52695bf 100644 --- a/util.py +++ b/util.py @@ -2,11 +2,21 @@ import os import errno +from async import ThreadPool + try: import hashlib except ImportError: import sha +#{ Globals + +# A pool distributing tasks, initially with zero threads, hence everything +# will be handled in the main thread +pool = ThreadPool(0) + +#} END globals + #{ Aliases From 05cee2eb6b35d5216d4dd34bed50cdc921668cd4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 22:22:33 +0200 Subject: [PATCH 0006/1790] Added multi-threading performance tests which show that, during compression and decompression, it is not a tiny bit faster than without, which is due to the GIL and even separately locked zlib module implementations. The only way to make this faster update the resepctive c modules to drop the gil, and their own locks where possible --- ext/async | 2 +- test/performance/test_stream.py | 102 +++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/ext/async b/ext/async index 164bb702e..8cfa2542e 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 164bb702e3871ab30341a714ef517fb58cd76772 +Subproject commit 8cfa2542ed623627b5e2e91072368209710e9370 diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 8916d3e58..298207963 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -3,6 +3,14 @@ from lib import TestBigRepoR from gitdb.db import * from gitdb.stream import * +from gitdb.util import pool +from gitdb.typ import str_blob_type +from gitdb.fun import chunk_size + +from async import ( + IteratorReader, + ChannelThreadTask, + ) from cStringIO import StringIO from time import time @@ -19,6 +27,30 @@ ) +#{ Utilities +def read_chunked_stream(stream): + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + +class TestStreamReader(ChannelThreadTask): + """Expects input streams and reads them in chunks. It will read one at a time, + requireing a queue chunk of size 1""" + def __init__(self, *args): + super(TestStreamReader, self).__init__(*args) + self.fun = read_chunked_stream + self.max_chunksize = 1 + + +#} END utilities + class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it @@ -27,7 +59,9 @@ class TestObjDBPerformance(TestBigRepoR): @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' print >> sys.stderr, "Creating %s data ..." % desc @@ -35,6 +69,7 @@ def test_large_data_streaming(self, path): size, stream = make_memory_file(self.large_data_size_bytes, randomize) elapsed = time() - st print >> sys.stderr, "Done (in %f s)" % elapsed + string_ios.append(stream) # writing - due to the compression it will seem faster than it is st = time() @@ -78,7 +113,70 @@ def test_large_data_streaming(self, path): cs_kib = cs / 1000 print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) - # del db file so git has something to do + # del db file so we keep something to do os.remove(db_file) - # END for each randomization factor + + + # multi-threaded mode + # want two, should be supported by most of todays cpus + pool.set_size(2) + total_kib = 0 + nsios = len(string_ios) + for stream in string_ios: + stream.seek(0) + total_kib += len(stream.getvalue()) / 1000 + # END rewind + + def istream_iter(): + for stream in string_ios: + stream.seek(0) + yield IStream(str_blob_type, len(stream.getvalue()), stream) + # END for each stream + # END util + + # write multiple objects at once, involving concurrent compression + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + st = time() + istreams = istream_reader.read(nsios) + assert len(istreams) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + + # decompress multiple at once, by reading them + istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # store the files, and read them back. For the reading, we use a task + # as well which is chunked into one item per task. Reading all will + # very quickly result in two threads handling two bytestreams of + # chained compression/decompression streams + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + + istream_to_sha = lambda items: [ i.sha for i in items ] + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = ldb.stream_async(istream_reader) + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) From 0ef86550179b9bb9e29ecccdccd586713b9d1752 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 13 Jun 2010 13:44:10 +0200 Subject: [PATCH 0007/1790] Now using the async zlib module if it is available to allow performance gains through multi-threading. --- ext/async | 2 +- fun.py | 2 +- stream.py | 5 +++-- test/lib.py | 2 +- test/performance/test_stream.py | 13 +++++++++---- test/test_stream.py | 2 +- util.py | 6 ++++++ 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ext/async b/ext/async index 8cfa2542e..77bf7bef7 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 8cfa2542ed623627b5e2e91072368209710e9370 +Subproject commit 77bf7bef748b019a3a59693cef6d955f74b358ad diff --git a/fun.py b/fun.py index 80b0f41b6..c766f8e09 100644 --- a/fun.py +++ b/fun.py @@ -6,7 +6,7 @@ BadObjectType ) -import zlib +from util import zlib decompressobj = zlib.decompressobj diff --git a/stream.py b/stream.py index 309df28c4..10bc8901a 100644 --- a/stream.py +++ b/stream.py @@ -1,4 +1,4 @@ -import zlib + from cStringIO import StringIO import errno @@ -7,7 +7,8 @@ to_bin_sha, make_sha, write, - close + close, + zlib ) __all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', diff --git a/test/lib.py b/test/lib.py index fc0982bcb..723958ca1 100644 --- a/test/lib.py +++ b/test/lib.py @@ -3,9 +3,9 @@ OStream, ) from gitdb.stream import Sha1Writer +from gitdb.util import zlib import sys -import zlib import random from array import array from cStringIO import StringIO diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 298207963..5de463ee2 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -1,5 +1,4 @@ """Performance data streaming performance""" - from lib import TestBigRepoR from gitdb.db import * from gitdb.stream import * @@ -53,7 +52,7 @@ def __init__(self, *args): class TestObjDBPerformance(TestBigRepoR): - large_data_size_bytes = 1000*1000*10 # some MiB should do it + large_data_size_bytes = 1000*1000*50 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB @with_rw_directory @@ -147,19 +146,22 @@ def istream_iter(): print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - # decompress multiple at once, by reading them + # chunk size is not important as the stream will not really be decompressed + + # until its read istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) ostream_reader = ldb.stream_async(istream_reader) chunk_task = TestStreamReader(ostream_reader, "chunker", None) output_reader = pool.add_task(chunk_task) + output_reader.task().max_chunksize = 1 st = time() assert len(output_reader.read(nsios)) == nsios elapsed = time() - st - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) # store the files, and read them back. For the reading, we use a task # as well which is chunked into one item per task. Reading all will @@ -167,13 +169,16 @@ def istream_iter(): # chained compression/decompression streams reader = IteratorReader(istream_iter()) istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 istream_to_sha = lambda items: [ i.sha for i in items ] istream_reader.set_post_cb(istream_to_sha) ostream_reader = ldb.stream_async(istream_reader) + chunk_task = TestStreamReader(ostream_reader, "chunker", None) output_reader = pool.add_task(chunk_task) + output_reader.max_chunksize = 1 st = time() assert len(output_reader.read(nsios)) == nsios diff --git a/test/test_stream.py b/test/test_stream.py index af7fdc35a..4f022286e 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -13,6 +13,7 @@ NULL_HEX_SHA ) +from gitdb.util import zlib from gitdb.typ import ( str_blob_type ) @@ -20,7 +21,6 @@ from cStringIO import StringIO import tempfile import os -import zlib diff --git a/util.py b/util.py index fd52695bf..6b8862472 100644 --- a/util.py +++ b/util.py @@ -2,6 +2,12 @@ import os import errno +try: + import async.mod.zlib as zlib +except ImportError: + import zlib +# END try async zlib + from async import ThreadPool try: From 97a17dc4f3af188c8c00b0b265f17c26c9c96ddc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jun 2010 18:16:31 +0200 Subject: [PATCH 0008/1790] Made the db module a package to have enough room for expansion --- db/__init__.py | 7 ++ db.py => db/base.py | 218 +------------------------------------------- db/git.py | 33 +++++++ db/loose.py | 186 +++++++++++++++++++++++++++++++++++++ db/pack.py | 11 +++ db/ref.py | 7 ++ ext/async | 2 +- 7 files changed, 250 insertions(+), 214 deletions(-) create mode 100644 db/__init__.py rename db.py => db/base.py (50%) create mode 100644 db/git.py create mode 100644 db/loose.py create mode 100644 db/pack.py create mode 100644 db/ref.py diff --git a/db/__init__.py b/db/__init__.py new file mode 100644 index 000000000..05d9b21b3 --- /dev/null +++ b/db/__init__.py @@ -0,0 +1,7 @@ + +from base import * +from loose import * +from pack import * +from git import * +from ref import * + diff --git a/db.py b/db/base.py similarity index 50% rename from db.py rename to db/base.py index 8107fee25..2cda0ea0a 100644 --- a/db.py +++ b/db/base.py @@ -1,50 +1,15 @@ """Contains implementations of database retrieveing objects""" -from exc import ( - InvalidDBRoot, - BadObject, - BadObjectType - ) - -from stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - Sha1Writer, - OStream, - OInfo - ) - -from util import ( +from gitdb.util import ( pool, - ENOENT, - to_hex_sha, - exists, - hex_to_bin, - isdir, - mkdir, - rename, - dirname, join ) -from fun import ( - chunk_size, - loose_object_header_info, - write_object, - stream_copy - ) - - from async import ( ChannelThreadTask ) -import tempfile -import mmap -import os - -__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', - 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB') class ObjectDBR(object): @@ -104,6 +69,7 @@ def stream_async(self, reader): #} END query interface + class ObjectDBW(object): """Defines an interface to create objects in the database""" @@ -183,181 +149,7 @@ def db_path(self, rela_path): return join(self._root_path, rela_path) #} END interface - - -class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): - """A database which operates on loose object files""" - - # CONFIGURATION - # chunks in which data will be copied between streams - stream_chunk_size = chunk_size - - - def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) - self._hexsha_to_file = dict() - # Additional Flags - might be set to 0 after the first failure - # Depending on the root, this might work for some mounts, for others not, which - # is why it is per instance - self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface - def object_path(self, hexsha): - """ - :return: path at which the object with the given hexsha would be stored, - relative to the database root""" - return join(hexsha[:2], hexsha[2:]) - - def readable_db_object_path(self, hexsha): - """ - :return: readable object path to the object identified by hexsha - :raise BadObject: If the object file does not exist""" - try: - return self._hexsha_to_file[hexsha] - except KeyError: - pass - # END ignore cache misses - - # try filesystem - path = self.db_path(self.object_path(hexsha)) - if exists(path): - self._hexsha_to_file[hexsha] = path - return path - # END handle cache - raise BadObject(hexsha) - - #} END interface - - def _map_loose_object(self, sha): - """ - :return: memory map of that file to allow random read access - :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(to_hex_sha(sha))) - try: - fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) - except OSError,e: - if e.errno != ENOENT: - # try again without noatime - try: - fd = os.open(db_path, os.O_RDONLY) - except OSError: - raise BadObject(to_hex_sha(sha)) - # didn't work because of our flag, don't try it again - self._fd_open_flags = 0 - else: - raise BadObject(to_hex_sha(sha)) - # END handle error - # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed - - def set_ostream(self, stream): - """:raise TypeError: if the stream does not support the Sha1Writer interface""" - if stream is not None and not isinstance(stream, Sha1Writer): - raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) - - def info(self, sha): - m = self._map_loose_object(sha) - try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) - finally: - m.close() - # END assure release of system resources - - def stream(self, sha): - m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) - return OStream(sha, type, size, stream) - - def has_object(self, sha): - try: - self.readable_db_object_path(to_hex_sha(sha)) - return True - except BadObject: - return False - # END check existance - - def store(self, istream): - """note: The sha we produce will be hex by nature""" - tmp_path = None - writer = self.ostream() - if writer is None: - # open a tmp file to write the data to - fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - writer = FDCompressedSha1Writer(fd) - # END handle custom writer - - try: - try: - if istream.sha is not None: - stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) - else: - # write object with header, we have to make a new one - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) - # END handle direct stream copies - except: - if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - finally: - if tmp_path: - writer.close() - # END assure target stream is closed - - sha = istream.sha or writer.sha(as_hex=True) - - if tmp_path: - obj_path = self.db_path(self.object_path(sha)) - obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) - # END handle destination directory - rename(tmp_path, obj_path) - # END handle dry_run - - istream.sha = sha - return istream - - -class PackedDB(FileDBBase, ObjectDBR): - """A database operating on a set of object packs""" - - + class CompoundDB(ObjectDBR): """A database which delegates calls to sub-databases""" - - -class ReferenceDB(CompoundDB): - """A database consisting of database referred to in a file""" - - -#class GitObjectDB(CompoundDB, ObjectDBW): -class GitObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file - - It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations - """ - def __init__(self, root_path, git): - """Initialize this instance with the root and a git command""" - super(GitObjectDB, self).__init__(root_path) - self._git = git - - def info(self, sha): - t = self._git.get_object_header(sha) - return OInfo(*t) - - def stream(self, sha): - """For now, all lookup is done by git itself""" - t = self._git.stream_object_data(sha) - return OStream(*t) - + # TODO diff --git a/db/git.py b/db/git.py new file mode 100644 index 000000000..d2477d7b1 --- /dev/null +++ b/db/git.py @@ -0,0 +1,33 @@ + +from gitdb.stream import ( + OInfo, + OStream + ) + +from loose import LooseObjectDB + +__all__ = ('GitObjectDB', ) + +#class GitObjectDB(CompoundDB, ObjectDBW): +class GitObjectDB(LooseObjectDB): + """A database representing the default git object store, which includes loose + objects, pack files and an alternates file + + It will create objects only in the loose object database. + :note: for now, we use the git command to do all the lookup, just until he + have packs and the other implementations + """ + def __init__(self, root_path, git): + """Initialize this instance with the root and a git command""" + super(GitObjectDB, self).__init__(root_path) + self._git = git + + def info(self, sha): + t = self._git.get_object_header(sha) + return OInfo(*t) + + def stream(self, sha): + """For now, all lookup is done by git itself""" + t = self._git.stream_object_data(sha) + return OStream(*t) + diff --git a/db/loose.py b/db/loose.py new file mode 100644 index 000000000..37aad8c6f --- /dev/null +++ b/db/loose.py @@ -0,0 +1,186 @@ +from base import ( + FileDBBase, + ObjectDBR, + ObjectDBW + ) + + +from gitdb.exc import ( + InvalidDBRoot, + BadObject, + ) + +from gitdb.stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + Sha1Writer, + OStream, + OInfo + ) + +from gitdb.util import ( + ENOENT, + to_hex_sha, + exists, + isdir, + mkdir, + rename, + dirname, + join + ) + +from gitdb.fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) + +import tempfile +import mmap +import os + + +__all__ = ( 'LooseObjectDB', ) + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(to_hex_sha(sha))) + try: + fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + raise BadObject(to_hex_sha(sha)) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(to_hex_sha(sha)) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(to_hex_sha(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + writer = FDCompressedSha1Writer(fd) + # END handle custom writer + + try: + try: + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + + sha = istream.sha or writer.sha(as_hex=True) + + if tmp_path: + obj_path = self.db_path(self.object_path(sha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + rename(tmp_path, obj_path) + # END handle dry_run + + istream.sha = sha + return istream + diff --git a/db/pack.py b/db/pack.py new file mode 100644 index 000000000..e57241a32 --- /dev/null +++ b/db/pack.py @@ -0,0 +1,11 @@ +"""Module containing a database to deal with packs""" +from base import ( + FileDBBase, + ObjectDBR + ) + +__all__ = ('PackedDB', ) + +class PackedDB(FileDBBase, ObjectDBR): + """A database operating on a set of object packs""" + diff --git a/db/ref.py b/db/ref.py new file mode 100644 index 000000000..2c63884bc --- /dev/null +++ b/db/ref.py @@ -0,0 +1,7 @@ +from base import CompoundDB + +__all__ = ('CompoundDB', ) + +class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" + diff --git a/ext/async b/ext/async index 77bf7bef7..af0040b0f 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 77bf7bef748b019a3a59693cef6d955f74b358ad +Subproject commit af0040b0f3c6ede3be5b2d6bc69f6ea5ac53c36c From 133988a9b53400810d2baea9cc817c67dd1577a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jun 2010 18:27:29 +0200 Subject: [PATCH 0009/1790] update db test to allow testing of individual database types, giving more fine-grained control over testing the db packge --- test/db/__init__.py | 0 test/{test_db.py => db/lib.py} | 31 ++++++++++++++----------------- test/db/test_loose.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 17 deletions(-) create mode 100644 test/db/__init__.py rename test/{test_db.py => db/lib.py} (93%) create mode 100644 test/db/test_loose.py diff --git a/test/db/__init__.py b/test/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_db.py b/test/db/lib.py similarity index 93% rename from test/test_db.py rename to test/db/lib.py index 7ba770f48..8d61e677b 100644 --- a/test/test_db.py +++ b/test/db/lib.py @@ -1,22 +1,29 @@ -"""Test for object db""" -from lib import ( +"""Base classes for object db testing""" +from gitdb.test.lib import ( with_rw_directory, ZippedStoreShaWriter, TestBase ) -from gitdb import * -from gitdb.stream import Sha1Writer +from gitdb.stream import ( + Sha1Writer, + IStream, + OStream, + OInfo + ) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type from async import IteratorReader from cStringIO import StringIO -import os + + +__all__ = ('TestDBBase', 'with_rw_directory' ) -class TestDB(TestBase): - """Test the different db class implementations""" +class TestDBBase(TestBase): + """Base class providing testing routines on databases""" # data two_lines = "1234\nhello world" @@ -166,13 +173,3 @@ def istream_generator(offset=0, ni=ni): assert count == nni - - - @with_rw_directory - def test_writing(self, path): - ldb = LooseObjectDB(path) - - # write data - self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) - diff --git a/test/db/test_loose.py b/test/db/test_loose.py new file mode 100644 index 000000000..70cd7742c --- /dev/null +++ b/test/db/test_loose.py @@ -0,0 +1,13 @@ +from lib import * +from gitdb.db import LooseObjectDB + +class TestLooseDB(TestDBBase): + + @with_rw_directory + def test_writing(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) + From 937d592ad08cff4bcb675a96d62534c65cc8015c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 01:06:01 +0200 Subject: [PATCH 0010/1790] Added LockedFD class including test, it moved 'down' from git-python --- test/test_util.py | 78 ++++++++++++++++++++++++- util.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index 5aac5b84b..2272b53e5 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,9 +1,13 @@ """Test for object db""" +import tempfile +import os + from lib import TestBase from gitdb.util import ( to_hex_sha, to_bin_sha, - NULL_HEX_SHA + NULL_HEX_SHA, + LockedFD ) @@ -12,4 +16,76 @@ def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + fp = open(file_path, "rb") + try: + assert fp.read() == data + finally: + fp.close() + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + my_file_fp = open(my_file, "wb") + my_file_fp.write(orig_data) + my_file_fp.close() + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.failUnlessRaises(AssertionError, lfd.rollback) + self.failUnlessRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesnt fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.failUnlessRaises(IOError, olfd.open) + + wfdstream.write(new_data) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup diff --git a/util.py b/util.py index 6b8862472..291855630 100644 --- a/util.py +++ b/util.py @@ -1,5 +1,6 @@ import binascii import os +import sys import errno try: @@ -89,3 +90,146 @@ def to_bin_sha(sha): #} END routines + +#{ Utilities + + +class FDStreamWrapper(object): + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + +class LockedFD(object): + """This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + :note: with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """Open the file descriptor for reading or writing, both in binary mode. + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + :note: must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exlusive here, as we obtained the lock anyway + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + # END open descriptor for reading + + if stream: + return FDStreamWrapper(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + :note: can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + :note: can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if os.path.isfile(self._filepath): + os.remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + else: + # just delete the file so far, we failed + os.remove(lockfile) + # END successful handling + +#} END utilities From f50643ff166180d3a048ff55422d84e631e831b1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 01:07:32 +0200 Subject: [PATCH 0011/1790] Added basic frame for packfile implementation and testing, as well as the testing of the corresponding PackedDB. It wants to be filled out now, but the design not yet done either actually --- db/pack.py | 33 ++++++++++++++++ exc.py | 2 + pack.py | 1 + test/db/lib.py | 4 +- test/db/test_pack.py | 12 ++++++ ...fdfa9e156ab73caae3b6da867192221f2089c2.idx | Bin 0 -> 1912 bytes ...dfa9e156ab73caae3b6da867192221f2089c2.pack | Bin 0 -> 51875 bytes test/lib.py | 37 ++++++++++++++++++ test/test_pack.py | 16 ++++++++ 9 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 pack.py create mode 100644 test/db/test_pack.py create mode 100644 test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx create mode 100644 test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack create mode 100644 test/test_pack.py diff --git a/db/pack.py b/db/pack.py index e57241a32..a850e0fb2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -4,8 +4,41 @@ ObjectDBR ) +from gitdb.exc import ( + UnsupportedOperation, + ) + __all__ = ('PackedDB', ) class PackedDB(FileDBBase, ObjectDBR): """A database operating on a set of object packs""" + def __init__(self, root_path): + super(PackedDB, self).__init__(root_path) + + + #{ Object DB Read + + def has_object(self, sha): + raise NotImplementedError() + + def info(self, sha): + raise NotImplementedError() + + def stream(self, sha): + raise NotImplementedError() + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + def store_async(self, reader): + raise NotImplementedError() + + #} END object db write diff --git a/exc.py b/exc.py index 3eaf5777d..482726e3b 100644 --- a/exc.py +++ b/exc.py @@ -12,3 +12,5 @@ class BadObject(ODBError): class BadObjectType(ODBError): """The object had an unsupported type""" +class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/pack.py b/pack.py new file mode 100644 index 000000000..676fa26c5 --- /dev/null +++ b/pack.py @@ -0,0 +1 @@ +"""Contains PackIndex and PackFile implementations""" diff --git a/test/db/lib.py b/test/db/lib.py index 8d61e677b..738eeb851 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -1,6 +1,7 @@ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, + with_packs, ZippedStoreShaWriter, TestBase ) @@ -16,11 +17,10 @@ from gitdb.typ import str_blob_type from async import IteratorReader - from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' ) class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_pack.py b/test/db/test_pack.py new file mode 100644 index 000000000..29b348876 --- /dev/null +++ b/test/db/test_pack.py @@ -0,0 +1,12 @@ +from lib import * +from gitdb.db import PackedDB + +class TestPackDB(TestDBBase): + + @with_rw_directory + @with_packs + def test_writing(self, path): + ldb = PackedDB(path) + # TODO + + diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx new file mode 100644 index 0000000000000000000000000000000000000000..fda5969bc69c10781d3159ecfc6a3caf36d3eea9 GIT binary patch literal 1912 zcmexg;-AdGz`z8=t%7h5R~Q zTXcX;mTl)hi_XvCD|Od)tJiK`87aAUf}`>E=V$DUY<600OIVw9x2C82n)#~P2M@Ja zwOIbMP34ktToN=Rxv~G6@XomAwC?=}9jBkO$(>>z`H-WFX~s6!!YzA>qI^uhIo^J| z?vlaQi2geQIj$C4F3k3sz99UGVC2uTnjZyzSJz2M-S_Pckk%}5U!fXnYJ9AH+OGu-(2 zw2i8TzvWXCk`_i)Jiqg$`DTLNnLou=p&UZzJav6maeY%g(W3UP=ufKhMAMBQBe=KE zJ1zE?^+QYJ?e~j*PoA-?_@BJT9S&UL4_P#nPuL6x5;q0p)nH(}Y$K7@A^B4QY zKFoO;LE2$LFBol?K3 zi)vlv$=yb0{+D?ki{h$wQz{2t5R^_?jnqvlONmddNX(5LTPZTs!na$V15Z&z(BqE8Ww(Zh%a!T)|W_!#O7_n1U3oD_JRosc_f1Thc=bDcsLm_cg&T4phZO^m^jPta0JV+A8E zvs$ok?@62+r$yBZ-aJZ?nkyUQ86c_*wSGfbtxfvQ6Xw1C~%USqch}(FGMK33|-WamcqvZ$n$kNsI{a7G*SUdu%VrNl}1Hc#tcwRkVbb{?_%)vi{yF2qOM zdf{c%x}@)?Kdg!$Qg35^ml1fJos3IP!!Qs)_ddm3Akp~SF+x=(7Ercufw4VPvPfJ7 zSAyHOM?h?!-g{#g0SaVW5oM<(`%Ih;Ud3vlRs&(R8Y59gbLCyO0jxn&!02*nQd7>} z*~&4uK#{yEqj$cb7`~0KzQg-%TRObJ8?U@|c)l#-d&qve_R@dm+Semk7rpg_m|(&K zqIuAD84v4UwRt^%f}PVG=3vana@nL!U3DtIPk$JiJ87m=5}F!#oSlrzN&_(%h1cdO ze!3|LQ*)Wziil7U5qyYT{%O;8k|C3UKE8wKLKiOF9nSZi(<&kuBaJm88s;hDpcQ&4 zBWI3AtGxD1nOa7!%g_p@ zm5A%#I@J-wfD*0ZQBdVwAZ}@=QsHgMslWkV-RN?G$75=9!}>fGUv9fl#wTE03MRB< z20E%KC2KpTwp#yMlXt8~1YcbaC)l3D!{}0O-I6C4E?x0w3~lb_R#RwOSW$sbgOGdp z+ujp^mvn>8PYT@^-w<*rdAP1|-jh4Oq-#t0*VitP54%c*{gNJdoSo6nO2jY_2Jm~H zVs5KK*ZkSE%_1TQB7)B#lVq|9v`J~RY#-lxkOe&mo@V&?e#~L&Oa#<(ZjFy79QcdhijCMQ}%?^UqI2&>R{s9L=x^xaohQ{W^; z>m|p7@#v&fWKFNP#pzZH=`A^i=NiilZ|9r}Ji{xFm1J^KV*U!97T8F-wXjloI-F%(7j`HDw31!3BkWRgjXh~P#9KjI~M>5O%fl4d~v z-a+lwh1+}2;k3#OCbHs4)~KWo5_P~REhZPx#DSgm)@YJu5ErVP8+cF34Mt0^$@L}? zp;qJ)C(SkvtL;p12Y=&3Fb*j>XK)CNiUUc~Yfmcd`xxaRxIQ`*w6w8Qcv_yK?Gd=+0p zp?x!v8+e?ZjLk{|F%(Akd5XKM3Zwa-q=*P^MDQ6TxpSuz+ewBr#rpORMHj9Fx8FJ6 zX_W{_DveS}BpzKr8r7)P7}k+1OV$}Fqp?D=B~;1{wD3KWLOXb^dgDoO=K`1BMXOCO zqK?>06Mp<*Wf%;1(P>FmU@rzQxe|OF6_1g)RL*DdW5$q=(5KD|Nk z1-uo-S5#4>Gea;rE>bhmO5PZBP6nlkZPeKkqQrI4j)FL_a~+`NYHXA`D`!^|B_~AB zoc(GwW@VL?DIldk-m2({4J|oZ<6UGWLqMrxNQ-q|C7Z}=hXQY7YaL#1wKRB!7u>Nn zc(|7FF}Ob+Ew!&Rwfzy4b~-L|gok_Sr4-ZM%b3)^(&CN#l_7Pkw83e|QpZV%JNQ)> z0kDm6oaqAi*-D0W-d5@ru)=JsWfXib-<;uSljkme0FabvD47{}oSl%tN(3{aj?RW98hyMsGaDr~(tK7+v%0citJqs-im09PW3bK%ytcJec)sRT;1Qm1!(8C*oZ4IC`mh&YK8{qjdl=QMoSB@I zFi0z<=<1eQr~Z`|FI>+IzG9{o4wY^qA+5I67_ ze-Z%S){#zt?5~KOrT1_ZU#TI22%jBzoSjh1PQx$|y!$Iw zJ;9-s^K4QO0&ziH`GU2*PF5s#v0hrxFK$<(d@h=5kb)zS9x1%qnozzeQhjh ztSBEXc9khx({*jRR|_Tz0!?o^S9QLn+84d)E893**L{ViDE7KV>k3_%M&T-BOz^x0 zj_?RiIAe%#JFpyu()|)meBaqP-$79}W#86a-@sPywN@FOSu*Qi-)a!|MWfQ-Z|^< z{LXz8_wH3wQ4v#cN0Z@tm0pZcE{of zNZ_cu9H)Tl3HE+nb)_a_voEL=?LRc-1xI7N;CLXx(cR|99ea0Owx!kr;LG#-Qa%|} zbJ)!M)G^be0Pc6t5!XbK!wD|dS8GNNJa-cJ&s5n9{t?r|A;?}xEcj+<92U?&YZL9# zezA(=zSAQ08WU{zx8p&us(q@eVn>gigjwlZoH%A?3OlNA3bb|HO+!q-?!0fG#1C9b z(jWY_FPZw(W2Prg>KGdXrRK)*pMU;0_RvDxwIw-|{gd@48Zm);BHhp!B+wFnDO6TQ z2gXr}#3ue6%3!-{LSE@@+$SdX7a-52N9>y zKe;sI>VrlgL6d7Fa<*q=Goq6lEu03}U%a^B^?J2I0mD5p%$U7f-1Ee@6tj@R*ab*? z6ley6h7-YR_Q?f@#4`Q)GOVVbs%_so;S$(#cZFbZ2#{c>$(lm!v+t8DRWIn6*GLw1 zM+(sh0f9pMAYmjp0fcXRF<;(PoV~I^*++Sd?F1r9cYiel3_#;S*tpwV?~An;lXT@y z`#dR?!>!~kpH@Q<5uXj+@+xrde6c5Awr{dbCMfokI|D)iERmHbT3Fqg<5<27X2kBF z+0Vx&Mnm8T2CK1r@w8!d|+Ms}Qzvs7dXUW@N$x-I*xOZ9ghurl&G=sspfVI2`W{cU@zi8Z>j{E46 zQ9)AL7bi?FP{4>o>di}tQhB^VQ)<7~==w+g_94qt1Oy0A7!d_m1gw>wRbCp%bojQ; zJM8k^YE~Yrft|E!2yiR{8fL6NwCryTd@i&-T@ad{uz0hz|8VeXg95&{Z^<_H5i1)w zqv5Dg@O|R9tzkaRkkJ?t8ViHFx`LIx!Jb+_=QD9+^G^Abh1UJKoPDbPj1Y^(fzuc2 z-1)=bn>{VRUUR>8j3bJjd3NxM_zgyl$7B3KTC-1+<$;(&K<|HWq`>y>g`_cGFz zLBp#mlJcSI?82w$M?pXaPk{@w`3nZT+!rty3W>oZp%G#yj=280l0)Kl;rm^gla7li zr9JwLWiW6rsCqK5?*VOI%R!Hv52aWuextZg7hMLiWm+a>}54!Jj zHdkh}WUMO&syEq2KzLo zC2n-9XiKIfqedaYkwBdSm);!)5@qI$3OL`-?0vJ85w!1Wm;bc4C!2wre#IR8-> zVH+9CCJu{COKqytmaZ;l_FJnIj!c4~v93rzu*~gS-?@LmEM8kWU6}YZUG1)W@?zKj zDFfwT5qlk zCB-ckYi~b;O1*DM@`=`J+SR?5r?#X2+fG82N!_6yZLYAdOm|fk3-#F6;GkS;cxPG1 zk-WF`ADR`WOc9%BI{ z34wyaNH`n@ETH*fCgUQW`Da^pz>a*o=GU-*6vjnkxd0xH@C3a}g6h;!iOiA%lD}(3 zl3aG>?rzgv%R>vDL3L2(famGTq(t6J@^h?Fup9wK>P3QiAqnnCm=_Tm2`LuZKw+`T z4YJ$V2eK!sZTB}dImRM50t^WUUkIFoV2N9IwD{2n;&OiFa+L_1b0~Pq^_wu3AX47E z`qC!l(=94XMVxc?AtzZCI}8my7^FM2iw7UN@jA_vJ#%zlRH0Fz_u`hs?eUC`fP{8u zpDowh-aTwpcyl)ezyCtKo7ugi5)2YrJ3;w*^tRuvT~olLY+Fwrg(H{OW&UPdw$NB` zqPapFTi5xzZzZy9C3iV5ND2g--&wHzBFKO-IH}ij%LUqR4??XT$z!J;*?R$@FU&AKunDtcaOE=jiHO!2tJYTC2NS{2@4SVcycuUU* zA=@FXcKEA5v%!qiliKhbL*p0HV|S7IZbr-JE4>!aek1L~XnmmdQj9suz5C^#c;h@a z7mjvBL4JlSgTsP5H~7ksUY{9RQ^B-rgQKhOOV$?ij~i~@>toQ+azM8p2WV{Cq4yKz zk}S-kXS;7Y@#r1f?6pdOlNRC|g`6LM!CQ%L;L%9t+IyO)kb19@krT*RB&84%ikeEV zkFr+@O7gU7asyg-Ht~viM!j0A1W3=H%B>k1Ns&xfF=&kEQV?W;FQhP1G7(y#4(5G7 z()2MRO0M6{BNmX)()`Q{-okNNm<-YWv6HF>uNOXmP)5fLd*uYqR{NKOJGwX*w^?W&DGo z|3I5F7wBYgDroCVCDo_K<7uz`6CD_vnSdlBp=n7RV3z1SZ`!A4yTl=W;^%}&q}`8y zs1FQ-CPLaMy+VNlH}=RXx3qPs=J)ex%!1Gw8V@H>=CFQg4pLtvJgd;P|K^ygP*(TV z#NmHAlCtLsY|FlTHdN*$FUzT!+7nEcQiXfl8Lb!StP>nc7J|4+idmjT3Qw|(e{-^k zVsJzx)|J+G`}%kHg}Tml+uCLF3Em5w_Bw3i$;eUQuDjfOCWZ&kwTj*N@+~4bhZFrF zQYrZ`%fH(9Z`7;GeFv-l{+v+AdM>kjmc~pTH{%)S7^T4SHdlo-uPxtkXOYD3D2Vhp zj=25xXCR{{5zu(>U`*+A(f48wC)hu5r(d4R(y0ry&#PBqoR36*FBcpJji5fE?t-o3 zV~xp=pBhf6rK)$D`sFogF_a*|pfe#0k=z-B~urFM;Gw=?6iN%ce| z0UTy9>dqyn)m4w)5PZSY<9Wvom{}Nh$aB( zRh2{S`a^?P+j=Eq~E_n$eeA}B!U z{%BBei*H=Z)MD1UWb3Uzl|vU-h!^-MB<+M?hj=J5>V4&m1W#bK#cY7~)}IODf29{3 zT+b*gFv*Vu%&}yQ7Jd*t@@{j^0WHrhaV-lua&(pwY%O+v=-+j^s`R0#q;2?hp$wEg zchPEt0w(9}Eh92>adOxDl`Fe8?+-Ym6Sd_>QR3Ux!^zJ^AZ@iL9u-v2iBBb~wSk zB;n|TecH5F$3lr*fW@bzGvnBjl5WjXZ@d6mB_)0K>ECLoTh1{NZ=He>nfykwaq$Gw zgY)2>D$EMPL*wQ4)a4G=eBoUxGx{MLH@$Mr*6yQac@RXh7cKTQF|W7(93`3l>4N57 z^YwCJX1zs+|6wRV#;{BF>M?|6fqbt*<8al9_rfx?NFPD+$3r!p9xx8}?J~4N&Txe+ z6$>|fE6UnHC9ZHVZh(>l!nPmX3_iLwnS8@&Bqm-(i`ne9STU5qV)(n^Z3E*IqW5FG zB=3{a!^)-dEXR1n zY58wG3Hi;b_WsKB4uggcPMoA4_N7U^1)4Fr`I4z-p|9fqWQs_cgUi(oyKhRw|UB`%5CFcelpDk+LA9#`AUB^a2IA35K8UZ?Zkh^rkya zdV9s?i~HV>(+RL0z#|D5KSFDI+s42`pSQoSY>HtPDn9qQJ|O&-JENuj*a7_S<5D)2 z(Mc&-vhlog@;W)5ct(pOxKdJb62s~i?XQ`9xU2>TcX!?{4|Sg#VzhWP!V~TX_1n82 zp$Wi6$+_bis0~(M#8VsPP4gHv0f}8s;UOMezcf2-a)+(7JeT7`kcQ}!-m z{)M3gt|4ysqLAQ=eC46fJlR(i^oH-QC#PAe1u0q!ONr(i6<}Lmv+ke+C_stBV~={# z?wePcboehhbQQb_IZNA%k*;VGWyv%-lEUWMVAHdI+$%AsmUoki&nemw4GB(LXu_?3 zZ^r+sIF1PsFgnMBOd34uk6FJ$e9#0O7P=Dwp~FC^pSH#XhvihvFU%uDp6T@U+%)Ki zL^vV5p!0@yjH%`4>Ig+WV_%r+`vc~7zxqZP6}Y<<9(f|r7vee7ro8b++JnHx+~2ev zkdl`{dr|W;r~ic6W`FcIER>%&pAfyN^BX>IMuUITD=6SnN2L^eFlDQW^elIMVunY? z8+Y1mjT_pJj9=Ni07Oq#6**T;Y2JQeqh?%^4r>2N8(NLa36BWrDE?E;9*5+-!{Bw^ z2CCvy;S2Nv1pw!*o{*pP*W5}w%@MXcOJsjNaXWn#VBkhf+2WaIYl4ijKS7 z%xyvV9lNyW3OC#Fn=Vr{jwV6rZ5TMgNMJbx35tF2NCF1XP?)=_6|`sk!1DsE9>0>E zcdw5ReIYNX=3PsJ%)O@9nfOL`pJ6IJ>tI-u!%mN&U~0K3DRn0P8tZq&4+%uHjCxzP_HG9Xu7o^XS%$l`*vmAh^6;!N?>u25gP~;a@kb z{DykLEE^$TAHy0W9T$A z385t3N`fBzz1Bltq`U%=ehry^0)m){H(I`2Spt(F)#0(5O*h|0Um;*diFppJWWUE z>u_-O&%GvDDs(l1!BsF4gAq@x%dID6c1U=#^q zjzduaVHQUtgPU7r7KSA`A9!Kq=rA=qgCOC+b3NpDP1f_roiuWkCReWfqK{sD?M{tqdQQerYUOxgH`60z z+aS4jPpW+{)=<%89sLyZrmQQ=V-?g%o*6~qro7GDx{Fm!qV*1AMn3+c(*z`xYg;|r z!BP_4EurFN{&wr{X@5l-z2>fwkUI8HDF|XKEE`#BFDtM)$%JUI1aI0H%agYXV$fL6 zjWKA&~qd#p2D zR)6${yZc>^CYi1Dj_l%vb0sei(6Se5PbSS17ISO&aQj>lH>=scan;-UV3%LB_nTCV zvi<1A;WMOE@jnS!)7M8Y(K&GP!N3`f^Ih(STYi~!vh2;T7$zQUejowJLWg;y&p zgC`993kb>&YKREH$1~w}WEFgh;(KdyA!3DquLU64n8PMN6whd+x7U$_1~6}il4y4{ z?gTg?2*Pr)ha2XnAFr40*$_#5Gn@E{UZ8;K@AWR_>Jz-Prt@J2<67{tSQ0HtMuA;N zO;mtY#aVW-ZOnbful7UXkeaTPUn^#4kPd=kdrR$y^0-rR&x{^Bc43cPX!I$prWYsx z{bm2rqOcu*9`IJ49Qhnl@O$x&HBBtv9mpKsH@Kbi^7calT<;3q%1w$eOq%op1z42` zXyfKBd7RQ~q@GS8g{P0x zW95T$U)b14e>}a^RQ){H5lt^pfU)WIm(z8_-rPU>BEpols^3(V(}G>J>vF($eZ0o; z)>Mts(V2$}njxZ76^CPKxh%9798a05AX7iOAh2V4cv7^JBi>s5-0hAB*J;0w67|#4 zD=GkK-v3YxS>?H4`1#wc+os!vRj3Yw^g&{YC=wD4otEZz3nVP;`H&-iedN*k=)19& zUqa{CfGB{8c>nv9d~{_jyS8{nXQNuC!^S&t4**azOGX-VY|jD z4v$((qJUP1GL`}B**7v7=iz!|8MA8PCA9A!FqoCA0+^>4ChvU-OxftC&0@ZuXd{~f5;va8*k^z`5E;*WMjLv z`-*a>L*TMRpajEtYud;CQWw!a8kN2-RXP2RhI#w03-R#+uaHB+!Oryky51=vL(+SzFR?1 z;0!$xYTC`MJ;B&^jghevo|Q_9^1Ci(KhL2-`r95wvAa%w*qoL(4uWSMR9N$0F};Rd z+SWfsg3jia-Z6?7FuvmR&@Ca=!kcz7^735H z{bd$Na|(_4&8p2vYWCmlbrf^!rCr}r?sKT?5TMu=6P1_=gEw%M|ItWO?lWz_LS-wf z*<#GS@tfWS(MfOasK#{&%^Q%Xs#Y5m5YfeXG`j}ZoUM$nF|{`JPu%;tnQACGTuLeo ztV6gUCFKTda0@}gt~_X znAMhYWpw87?+40>bFfkktqo_lMm)OH(%1D5LjlJl8}sowgi;}mr2W-Hbz3enpRHZZ zd}tl)n|8!*VET3<|Gk62HD01SK3_L|O^O1d&82g$5+tW12E&Ha3WhVo`%@O_DJ8!O zR!I_>?>O*2qk^@pug$x8xAj}uAwfC^)^sOS*=W}dpZ0FP_?(GnzuvVCS(&W}cgnIl zDcBtO(pk4{%&RPXLU>9!)nJ>#3V~uj93J|z4{DTbLlbPDd!sB`^d!#hvX<)+LG7qD zDGHc~&uF}G*`&0@W!NhGg5FZ+nNph-3QZ(h`k_~W?}h2(ySsM1Z2j|Q!u5Wh!*KN~ zh=x{MuE{qWJTtsRO{|`&1b75C?o3pgVo=ZvT5plUAAj&;Tqf(CZxB0Ai2x?|;A|XP}$fwX@tEty~sQg(?_*}0&SDwheJVu2A;3CE*y6sJZhFu%~?8>o|@4eXR zNLAkC<%`4f(OZnGJ-zY>_bVEuBJ3we9G2&^E@yl{34151^ zh~4o5-T=cjAPOMWZghPS|EVLuwqfXC=;L2cwoB^#g;1+XhS^VL*X|27l1;2{Z)Zo! z_Ma4?7bw6t&@L#uJZ4%;?|vDx+Pn~F(W|Bv0Uf|#;HnONMw8N znJ@3vhBK=mbYa@8C3XH0e1lV6T71;5yvi20Q|ne66yWeocSCU6+55#En9HeSHZuHd z{O?x`3QvpjMu@f~MAKo-0r-H-02+%cnV^F^5^VnT9EnowiX3kI{0(<*!GjZZ}H`DG<-TA z&~j#gd4)XH%X>UANkSpKmXq~u z#C(lsh239}P;=&EJ8CB+< zIG3635MW~s&4Ho3%W}U*B#ZuX+xm_z_-)5W-zoL`i5ecEj0y#`!SWqMxGFZz*vUw{ zFdgK1=DO?NigE`R7fOK6BUF@X-Msg*@%zVfgXZE26o z7ZA%1lq#l01Vj?Sg$yP8vjsHkq$5k?2`6VaihgPz!XH}3)!OPnb324ys{?8|UdS8FA zp=eb8*z}N7}9rq3s8x20w0Td9x`G}3{NO9D$Hg1_u zZ0m$BvW~c{04P1L;*NJr-o&3Kv5qL@>_4>MM&YnN_3=nAa7BW?)fd_&^TM<+?Eo_H zh*jXlufB~Hf*Q0+pqvW= z?^MUs@;o=m%_s$9KG{gnJL+}S-9grwB>wO#liilDuSVCtLaWP~tzy)e=Am|GRcddK z6X(6r+?eE7!%LkT~@2I!Rx`9f>7?V-*K| zSQ7c(=bcMy5f&EGJYQhC&L#Yp(G0y^eNf4)LWTQvbndP>c=cFqYhqJYgD)$jJ0ylD;c(mr9{e{WP{(r9i~ zb$=(LZl{E4_|^@_4ft!HeKM~{c07u0pivZMtEauA64=C@gs**XBc(reXl?KKhoJyn9`++nNl{muN?bK(2k#bDD(u@z$H0ng zi`XZuvR&11TK^TePwi4E@6qz77bsxQ&uwO*srBOP=PN3-zNa{d$??z<0%R;2oYhF^ zt_(@Zy<1px8%c@b5kbO6IWvi={-Ot>$Sam|_@QoI4{nqNwU6<+3+(6}Dsn?jCO#cc zUkzz9fUr40Q1Cb}ittp`n^?hxZtC<=TT7yVrj$uDZG4^ZM;UbmtGTxy_CM&LrJ|_c z*-(=@>Rk^M!q_J3JwilAj}?<0+Ea&U9|@3&@a4}cbF(Z$%4D<3J(%|X;mzjr6*L!I zAxHwXv#c$W1~iYILt8vfbk1+#_H>rw3}jR&pi)2}ZmQf>YoyKkLA`VtcWk2}il#_z z>R>&cc#oX z65_Sz67;#B>(!U?SsUJzVb-qe4WojU6dX?nt3ta+UZBb}p?c^QA`*-0CENE#;UjiJzJTq!b`GIoQK09Fl+j`DN`?~h9_2X6$c(`@K z5u9AnP&>BRop9dy+1}RO%pYBTI=*M-MgnIYXyNd@!_a7O1^Z2XX7Wvc%2T~kW zY5z{iev`-$Qy-N);i>YKeX{CM9ztY;i)ln`>pu(yqzxYTEv=Orw$rsfyDgWad-$LZ z?ULP(`VBq_NpM?U44B5v3O5IhtNSIU)pY9paeK@5RP6s?6kusBuaVC6T^V8dTQFZT z4p$!DI=H;CQ7pj2Y5n|%wcM$(O?TIE6lHxF5*tGQ56w1ALgxg^S9qKOM*z70_;2)Y zu+k>8_9l`E6ik<(SL6{{`F4Exa8A{(7#iPAjkBCElmZuNa{%U@;+wM3)Wruj-I z%#bO0d@gU=e>_d|8XiZ4Bg@_imlU99fqT-ibdXUFD4?TL$ULFaWU_-hAbhyFftL?S zQId<7#0CIGSYCl(o;ew+GJN~;>9YlX7tET060AJ$?NfzA@fg5a!eB+2I4$i=%Da+n zNwDov{YYtmS@;)vQ{jp=s?N1R#-Vp5YSs2A>gwC(lpz!}AxkNV5-GVze@t>_s^HLJ z73~Ev?CKyk11ZcU#tj&TOIFsgB%v79dL3fzXM(mCmKddg zCkT|ili<3D*G_@@9@_5BabuqW66>M>M~o3TUCKRAb3{dmluTBaFq%5|wU_<$6meVk zTH%bk)`-}gV03xm<9?~mPoR;HZNvjcI8_MKv7^_Tai#Mk`n^Tlbq8%b3-)xndr^&W z98qxdJdqW9#01w>plrGh@_FhwlCK2&h0u0e=~&KyF|XNM=!!mB^_@+e_okp%oRVhe z_bqp;0opdJkk`%D_H?l1u=Y1$qyz5yZ18JtwGyYEh9XX@zsPO|>eDRNiaI61nKLb1 zHn58s@@OqC$74$7z!@c;*YPX@mo{54&!abw*=mjF(oJUXxSZVH)=lq%SMB!0as(zy zcHIK|GGWq{ugt;O1qUa~`NtA!!C%vLUjC)^3V582oC`QqUHktjL{U1SNK&FQm>DyO za;W4Gi4sYSnZYn-GzW*Iq=*tJ=Y$A_L}$^7l!{Uk)ss?0PB|Bn|K58JdwHJscU}MI zect!2>-v7~wf9>0y4St#wf10jbcnjTDq3DN4vE8I(LA^uDqBP`x!T+%G-I<^tDN_Z zZF}tw=wPg{sFLQzV6vzpp)aoC>WIx6l?3BdhOxjr|tW%{E&^nn$0u_^E$AfKSm1k2WkYygAi02`Tgl1Qn8( z7mMme;!rUxDw|8^u(jCUB8_)%yM|uOD0leg{qC@DFZqEvLwC? zdfh;(hd_AD-a*}()p~*zaYQUPBS-Um)+G^s0Z6OXhQWF|?zS#FjO^rxGPk&2 zm&EG>D##>;9l(H)jA9G~A0i`#G<4=KgUeuB>YG>m4msQaw z3Q-DW_HX{w>*_%NpfZtw_-}o3-z_$(Es&bGl9Rk}m9^(zen-M0e-el6jqzhrsIWn9 zPM^jG%i&K|cf}{lQqOm|uI2Uv4YQ~u3Wmd?QZarQd>|8)doZfE-;o6tw6SbQPV!-q=dXpu$!#uf}@?!R#= z?ZqWR!h3~37Tl4?{D4BJ-&VauFpeFvw#K7 zfEJHICj%^33S6RfU7>!K_vz@aVeRc_)3r0&0J$!P%LrsaGsuHyeWh8^?x@SjzRpUw zBhIR4N@12&jskK5h7HXi5*;xu^?TRVjp;4#XVz#8jhSavnr!&zGoU46D3C+L!i&x0 zaTjW?Kl178wagi=UbBBpO#u~RbucUvIT(INSlqTJ!=w#ozj@~mE3w6?*F>|<98W{V zI2{a`MfZe-wyc|2lY4}Z$Mb5~y_Y-oMKhE>0YWT>%wz;mS!@~;@z=Pj^MNm)$ek)a zJE!kggmY-oSF0RAjzjE1uD~GqArC#!`ZkJ|z1OF-{-%HTwz(QttH;FwJ)Wm0`N7(P zUe?YThLJhamgjFjE}=cEKPejlXm$Bo7TKE?fEa&TkK4rg@p%U}_e9#4=bsO?{WG%z z1@Jl;HZ+F*Y=06Nk=K^wydd2-ycs`}K)JSp<+H6J=@cs0)q!RRi%rEa{W;LfB4blt zv*KrROTbSDtw*n$#6B$UOeTn+1VIM_yAwN@&7mSM5Urv4>cCR=+narod(DWYDmK+K zY5+L_<3SB(GAI~!upgbq@I{)CW{!AWczf9164QLHKINKAF+1c6$`Ij);&@}Y92(DC z99}Iy^|`c`pDT{upEr3fz_vI75Ox}b<3s9-k6RB*FR074o4Po%eQDD!7sFlv!cBv?LTsPO z%W9p^vo76!7}cPtQG_ul{6`VM@BjwAmnSUc&pr)6 zy8mh`5SbPO=d9LRjYm%|Fne88TlypQ`=uB^03rYov@}tXId?2|zW4gz)FZi6Q=h26 zr_Wqx1vvl=5kn?%FoFQoe0y-J;-Oi{(jRZ*x;Gsk3F}-&0>pY4ZxY)Z!w;A6LXpz8 z<$)GQW2@BXop0(e#v&4>>407zcpxS;U+KIK8~ZXN;-jaU*fo&}wfScemt(fb)ooRHmR#Dn){XE*ru&LehzCr+YNix&V z4_YC4?}ARo$EMpy0{+arzH*zxy2-kEnz7H*rGy*~;y7HW2^!Duy}s++J|cR$_4aE_ zp?{re^M;Z(GocudxUxSJj%}VaFIa!{_8Jq}I_JhsC*4&dV`%B3vbz!iJr=|ECb6g# z1cUW0W^a_vsBAIabN{o^5jkRbJ=gXopvUp-1A+6Dj>{&NTg>R(L!C1|FIoAwEVip1 z0PvVV64^Hxcp^zr`6m_Sq*)OLL+P({U(f@SZL|TaQ!K~%m{JVop(2kxq0?H zbiY#kaZ!tq zx51)%$R+4ZHZ&8MP^g0C|74hG?%HjWqAa(}R=>ks{_*@_bgt>bQI<{ig@xIxal=Qy zN<^9!Naa>&OnzCWnouSb5|BX+;01A5ZP7O?@H}PF!Hsutdkl>egMas!HUnxbCXmSz zdhux9Z-Wb`+NuKgU8W9D>fg*;cmEt9#-XNp{|osgjJ1o2hj}n zk(ZsHmEdaJ27db+FE8~_MFD|8YYd9Ws_p5LDrQVsIWe=4xU6^p-*-_Q^sI5?D( zy{TkhCYJ*RCFqYBG6Z@fj=k}t>u%*sIP%ECja+%L!a2ENH2{eHKf#m&cIV2oySlSd z)HZ&8?fAA!&H5_<;{FYCVaT7T6vj}zs<`@y)KR@{4&9br+86-B{~Pl2h57^rXI%W_ zO3XSXYhI?mEjW`FVDdrQ765hs4MHolKg7-+6cEwa-HkkKhO06 zNFRWZI0-WwrA}N|Ub4Bcv;D~4c`H)4dd%@u02l*+fp$AIh9?q|hAJ1U96zMKSmOS| zo6qkycAREP$D-@F@BrxIvuRMNz_UahVW5-wD?Qsm~Fg$fD zGRzMk@Y6a2+Np4PYvp84|J6e!f3Akgkfvm_wJ^N}roI5Ci(v=SpuG(;w)4yZ#Qp5i(*aJkALAFpZ zVxy#$A(g6dbIMjL!Kq`$PUf~GaTI|<7QDrSMiZKubZCTl>FVu;%VH!y49>%yY8rca zV7@}W?hf=duxLc*d8`MC0;hg3h#8t}IXOzrzKQ)r4Drv;_v+hF0f1Nm$a7g1mG@Ve zq#QXmnL=1_fA|>XnPWW@_NtdFz71;+793Fr+r&>N zc?AKCAC1k10vF8VS0{8dgdH7!eR88iTg#w#qiw=UArG(CBnkzCK4`oOSHdvWyx~OV zuhlDFgjEbabVvo{SU`^91p8CT-Xw+>LTbhNud)}rXY(}HYf!z@TfepLiVHyE5EA+? zi|WVZjf2mT9TMBZ9q-M&Eau*=_%2yYW3V3}@CZR;aG)d_mOJIFNmAasIzw^hE~k&t z(iu8>JGKhta4JEDGAJhWuRom{Kt-|-L(wZ*Hm&w=4DW|~HI(K)pCs*z5+Hau`qJ6F ztpkS1@uu>EeB>JCqp4=!kF6Vcb7LOn@WExLtTF)x`)>wA4ILj+~9v zKC|tw$F-`N9~(>S02YU`6ewB?-0HYkiLIik9pS(M(PvAm&y@~pmdBzL7I;cu8l4U$ zKhTV%dD5uJ{4WaQ0GP^RF`B7P3aAHH|Lg^E4%+48S->W; z-!r0b-l?Gk7TzzDec^-@v2Uugr0D9iNb@}9z$9_?ezQwsK2L;dcVS=t|zezqEr zV);@yFXQkxCB=E(&d197TX78*W>rP0OQS1fZmq&NZq@B)DCOGC07%4Wuf>sNcdfB3gBs~!f)T-zJ*8qn(s^l)%vk;sCD zX2+Z zyZze%h={%nn@!@qPQ;&sR=;1~yE*%CcEUAIxmIvgY6yVn2@n#rG9!b%Cbc74Z)oY3 zA78u9xY3G426o#S1AxA;4KAHS!$4~n(oCynV^F8(`sV!ydRnk^N^a$fRAvK|fpBEt zojWUl-`~jiuL+gg8Bn%Sq4ebNx|-!%Ow*m@ARB!;FoHirUQDRmv}RoNJ=Vol4}<-_>(3vG&~9IrEgU4?YIZ% zb^r5(&_qWv%)Z(5+R00}C#v`cr{S|3(;Eskg#|P`fw!q(@izG%K31&FJm-J$oAI7Y zYInx|F6w)<3XsCR0Bwjkd;;T7U)Oh>zy++&nz*ubmFtzb*{+eK)c{7AhC!PS4$>X%%65A)%VzY1ZtES#iEGcy zvL$H=DIy$KkOE>rY5A5dJ_?r}RpA_I4oS7^s5zH~GCf|>giNIFc@L8E_Plp8!(?e* zOm_Qz{6iw3!wNGPV3K=GQd>`L+B#p}@1v6HPw|lB8L4XlhLAP4pRRiA$cCNME&mWzUvxA7Ck{zZr*P1t(ewAr2(X*Vf7jO zk`Am_qi;(V-BOV0+#cIn@?o3Dkz4QEc?TJYTx3*k=FY4)ubY$1<&OBj{6AM z!(#OH2?iLf7AJ@!qF=ex=kVpcO*7|tPx`%HeJ-6MAvsNpv@J~9vAKbBbj|4*jcYSS z?pB8EpJd2%Pm^;wp8AMkH+%8A8S8Q`Gw!r^v)8Pve|xF$IH1H~_H&sWUKn)p7GI?n zrq?Yk|Lg0S^G>}=ayL1E6c2CFcpKzhi+)+_6mb5o8(j2bURv|;2aSL>K&i`9qK`Xt z$nZ&z-S@DodTW(rsiy0nOBmJ)^l*=1kOM3@Ad`@u_4{NV@2bALe2Iee#kc-EKz*4$ggWQ?KFlq`|`5HRq|rfLu>7Qk(!@66gL&Wl>|X>%I)RlPBHP zPo4gGwifVCEZ-rZypk-KXfCop0&*s$cWoaWUnF8n(_NqOL>D0Vz7WVpHcH4r!iB%s z#Y$_3B6fB?kDK}R(btY+aVUV}y8<-`ith~GL*=_`j;(yZX9q>k;pOIW>MIkUyXnG? z$LWZqu0O7L_$ai6dZVFy&};>5yGm>aU=cg5<1q}bp9i$uA@R-s91i>Z)cHItE5z!` zgtocZdA%P1i4$~>(DTK7cSYpqwVm`4kj1avk2N2k4BMU@@O9@XTRQFdKdPnKxtdvm- zns8j^4{&^sK(Y^Bzr!0vs9l(C`}Q^NY1E|LQ(Q_`Rm8yc)Pw|OfW-@g9F5B0y_a&S zW%TDcQyp*T&vy^?`rvc=(ZmQq@O>hX$)X_Lw?*>jpMts#6>(QejY7V>!wx?)9{}V8 z!Q`j$j_f+NcABYUM$WoL8VA+>F~lyXsGbDm#OY%NpV~t&1rHev-`zA2VQH>YJ+JbD z_vce}TOM9BQv_f^lUw*C$R@Dgpr*&yO9DUqijkn|_+V zzAUYLtSa}%^&78^#7)Uq zS>I?5H}L)XYpabbU=GVS$6zv$fb?zi1`(SX^vKB|qdVWf)f=mNc?F^7up%z)d5Hm6 zv4Igw)^uM=KmVqIzHT{S4#zi#W-E)6lk82a9DQ;w*rqTW^K>S3j|Bq&o)4gIdS%Uh}PbNjx1i+X?K{XujL3`D{$@kLJ;yiYnwR!}?sJHp50@Xw{FO{~yE$RyHTerx*1;P3G9KBA7XV+NFD~vO&?NZI*eUW z&~vJc8?}g*QG8q8nCht6ln4-lF{Lsn{Gzhw_zq{~+kJ7e+RY4|cB{b;8*^jz>td}17f}Ew7-WC`f#%V-cfC^VDvQ-DD*n7QR#iP**ZCBX3#K_7b&w&vIp>eflcEj;W9?cTpJ$_xurk0E!_Sh_pTQ4QxgCR z#-8oX;txuS>F$2ROoz89qM~KmX$k0D`d(WMf$Tg?PMDYQtfb#u>xQqg^%Y z{?zRwXRHGV0!j#GwBc4oieh$yy3Omt=!9btQ!#CyKLP|192Hpq@>YK4mzw7;t1q`& ztk~r^SfFcL2IvLD&W2MRPh=9U51-L>c7EwIQKu_1^Ibor*tt)31B77Q1x1QDpDv4L z^A*L0Y%wy${^_cVS-HOEsn_f)@-pTq$Q` zkXktqWe<*HJdRHtq|bXgQQ66Sr&|6@^d$fbhToIqOU00&6`IZArAY~TMwhcHXK$un z3X1)?p~)Fq1uy`I&V4^7KaDP5{@0NH*K)qzb<4AVwG4YxPoJ7X34#v)PFQ%7yS?tG zn$^j<%f4it^CuE+JR1gN&;$9Q<-&FGm_QBVd&UpfhMxC*?m5@=?42s17>87E1!bJP ziWw(%&)>bKA=X+o=8bcd%$(8RfE2#(0U71re=u=Lo*&P>@Nk@a`J>L0(!yEv$F_hN zhvDm#6E$;PmJgVVh&nW>3@lSToUJ|!(BUzDP}b&0LQ1u1p^BS5b@egv8a{Lh&1V%K z<}?@sT3vq1K||JZv0ryP?TiD&2TY82)jzOFe{CMW0TmN)a4!HiWkzQ{D#g>SM55vY z_59B5m~}9{vPclAA+<>F$dmTvBVW|N)=Rg>So?^&&*~Od3J7pzfkR>mk0{)ZmB!`v z52T)UdHP4pk#$>ykS_^{v4RVE$c;(lD8v_guEAqBe)+7%;@0fBP3? z^~Ha_=GJx^KYIqtVs<5f;CM|4uSRWh?+C?)OkNE!33hV6k+m}SPWnPXj_1kAOcsla zj0k*m1HG>xpWT6byHLuTrdYao2-)R62jXR6WD`OjZ9xv<}y=`HD-MeqY z!%X7*op~{Uk^nstasY_oSK>-Ykyghyk$pP2nZun^s}oEmpf4*2pA@3>D;(Pmtmde>7}w!l@Q!$-b8eA%JXp_2fE=P|-#d;#X5?#cR-Hi^#RwP&Lp9u z)0}xz@d~l%{nbB_YwaSvtgbk1eGfozp_4}T6`pd+>gBKe%i%ItEi}&x93JwYzunuZ_YyS5?$p z6uY1NZm~#b>s!(`UqFf6UK5nmDx+~Le3#Wn8&*|iUm?Ax{Tn?J|_YCM005HEuD8JtLUYh#h%7pd!C)Ned}wVE8O59kSeJFW2aI!8ySX1D$kN#~&2crf_U_6XJOcEZBLVk+k_b`R-nZWEi(YX@c>{45Lw<-ziP@?xr5uoKDbL}=a}Bo zCKSNyfb$3=tlQ-!-3eAaSg^Tx-D^LZe}4Q>_9=LohQE7&DJ+IUrTU}ABH@rwo8NLO zo)ql4ap8&cSB)YqQj@Y6+M2Eo@_mO;J)HfDF29uQ7`B~M#N0jWlM{DwJE|t|)k625 zOT(Gl&+u*O#-5tl`m1^DQROlLBn^b}f@p+8bd3@(zN^hvylw6tv5Um{|n z{b5tkvl~bMvH%Rr!(io#t&VKPt4lI`?8XXLhF!Yx;NgSe7CrD(}3c4G5MwFkmv~?3!0ly#>QVem1N`HX)@}%JAP$F-#2S>Bj(&J2N?b{ zK{JQU%Q2VSeAs9|e$t$J^IOtU9ObJM(M1E$3!V&c6!ma7H|+D3vR~rf?AjGwaQwK& zwKD*ri}BzxD7@XprD4+ylJurEFaPZFJeI#gdy#mn7$7C^jpMwK?}jq%7jG+4tZCSz z9v>EPpe&__w89@y6ZvZN7RJz6zPsZaxBSMoaKmNd-H&>w%xh5qif?cRg-#$C`MF>H z(e;G&mgZ-Q!qj_ka*{cU!Yks?K|?HVl-ef# z+|E(QGi^t!D8S%RhQst{Vwfy`ZB~;#_#tHWa>qH7rh2|kp-SrkuSYcud-nd~$fTT_l#*UV)=%FLFnT~Uz(KI!*1vV+{Ff?c#w6Bub$h&A za(Tfrmv%jX)<@}SCFRFg)3fq>M_0-`c^>;n(qgCc{y!Z6W&rjHi+^pc=CV&(s`Zr? zcg>T(@>}Y%EV9JaPzEcwWhKZ4MK{0ibb8bLKwbr}WIqs=(ZD!T0>}ZcKrS$OK*0}9 z3EP!#EVoYNzPY*Vl#QC(NZ|p==qmsW_y?Qj=TGO~gWfTx%qw%vBZ)1~W<2kjA+aMr zxj-AB01x3;TKI3r%Z~nNOMJN*y=Qa|k#x`9dtvP^x;4m{u}zrV9BC@LDnYaC9QMI*PQpsbtwhpy6B}n^mfMeiBac= zGmGvwMeK@Mktkw*=lRQKfFbak056|zGP3y+CtdSKrIC2zTA|ETi{7em03q@b!H!cg zE8BkMY>oB4*UUGhI2&&i;_>$Kza6z+~UE@-t0HDtY zxc>0PPQew`-$UP~G(|p}$L*eZ@kh7XdS@PSnCDbmY5F!_|7adp)V0fXA z@jeygzIVs8y+*@9e#2> z+vQf%-;#U)6nFr9UmEIfAPsX#nwX<^$AE1X%xoz8`)pghgyBU1Bm(XRZMfmH88Q^k z5Er_`StREao4Fyh<6#ydZBA_dtFSF73guM7y`7*GA=28Cq~7^;o{h%01!^-cCcIHd zZAS?L^2rdLUyeU{vRO$gmGU~OX0f){Y?m#a?9`n?EtbEA3O^SS1d##p@v>Nq<@y`Z z4O_fOqJI`j(N$3jKG&tPp$YB5yQOeQQpusOYMK5qo7jnS2^wc*LxhK+M7Wsc=_l9* z`>~M%fZ6EO5c_WOtQ8)?qKfl1>CRV=hY1PfvLlm)p^~6vo&O0?n0<{;!w#x=z@BT1 zhQpSa)YQry699T}Cg6u06w2QD{n+~FN86G)2F|~)rQa_w_IIz@)(PX+J|4|j89G^i?LAvgSDfYWHKGpnjyL|5`q4{-xzAUOF>Rbbp7%(ZsD1(Y~c+3D7=xsgoux+EU?L;K@} zYJKfziGUK%Q$mY2ulQ;9U`LGi_n)(B?l1kdh-IjKdiHK3K(31#9w7FG+|-fsd|37D zjhal2Jo#g0H=rc&4GZsUJR1IEL}lOcv}D^CWi*Vn%21_*`LiBeRwW}-t$R}UGf^Tg zO47t9rC~pN`5n6j!jHf8kSq1@Mg&&Qk71N6tlRt6f3Cv@pF~BheZ1vWp%N)J33pS% zn@aOEYw7(940gP~R~{vWjf`8<;$jOhSb$-3{rvcA0{s!);4ioDd|`rib+Mkedc*Cm z%>ac1DD)EW*piwCn>IzQs=tzz@}D$40$v?s0T3Q+Dwu&h9ao=AIsU%P0=wzDzU;Pl zR~H%_hzAe?fC!h+v=zUOHst>Okn1-{TT&P+=6GD|G{6u62K}T0gMa_C$5`8ap?C-J z+d^-TrFG)R07wsjpuhmf1*EQGA87Z|>PUs$$rBCDCGni$@loq4plI6OL}1mRbmur#)3-sK8bP>N!j+-@bO*p}G|4W@$03@JtAjJvbX z{*4}uKUv;$e&vqf$4^h5159B7Q(u2NsK(ZkJg)xpGD!Bk8Dp{a!2&>v1Eh|1k?u?W zsoCG&l6P#NIrz`uqdUg{DIUJ=D){tNP#}!Z7~OViQT(KldgjbI%oBYoc6Gl1MpvMR z0ny@#Gu}xGp((mH>tq<{}g>nRmYid7DG>-Ig-e%dIN8`7;R~ z1$i{=kwO3yq%dSIl(hSSSJzvT5AM^Rqb|L>yz6|y@2^@vKS#I&AVJt3n@Q*8WYT4? zm#^60vgDe(fvh}s-rSBezs~^#aavO_V1IGPr`_R&PS!cL?84$9%s#xD9smfkSorI1 z;bkk!;a2Cgy70_TXQh{)bE%sBbcy%|fDt6J{1-vGK%>zsxir*e(du_ks~CD^j_l>j zCQ(GNFXS+p$gZiwyQ&*warvtDGxarjm7=Rh57l@I)q1?|sQJ~+;R5lkc1xH1-91yR zQ^#)m*s+nzmjNl3FU4>f{QpB@j^HOVTnaz0{;qZw=Uq0xmT_eCqQ$RY!n@^q`uy9j zf~V^K=T(|=*;u)y!k;}Yo%ilcdao9qN$DeJQv4r+fyG;1xOZ901q75FXsy}$(ERY) zeH&Dto^YdZngZ&oF`?9*uHXD1wm)i%ur<~dZDDkLrVWrzL`jw9ylK5Ht*a)jdSe~~YZC;6gv~J9aIHa;R-d%ZISfDn5-yAu7pu$INu(=6$pG@(4 zG)i^Uz}<|UsvWyuuM}QnHh}&h_$eB6@!UOkrgIxRL;j=!G3ubhr>0n=*oOdt6%zg| zCa)5ICL#4lQU(4!<8Dcbaz#>ozCK--B7b`{d@CLG)&_VzA@4`?V)0%0v|_B(#1l_3 zN34?AjXN0Y+cQctP#JKohQE;f&ND6hiW`gv@6_;@MHL1^{4yHiCRPBfzQ@ z#>UKZ++-G(&GY&sl6NFD@O?CjVNrMC!XF(4_@Pda#n|Q&Yn@`()dh1K^JBWjIPrl+ zfFAWq_{Wye+W+mJ8EVnH>^eEA_FsEK4!2g<0_!dR>5Bh)Zob%G6 zt#3x|`j7XkJl6njiZuXQYJT86@$;-tlT|m!)I15d_DVItI_ixiF2|d8-HPQb{2{Sj)PiB7V2B>|(iTNcHf+8J{o`OtQ$8~0(bsYy2ZKOxOVYFcfbbm-{F`-rBgY`&#I6p6fnla#(H}wbCbhpof_L09DXk=7JP5^ zNG;mWM3HI@iDhrvGuRz7LqFxOTl}nkr^|hyG1O=IpI{0)54LSp7+bptle}c|XbEMr zTXu4-Y_1BxqK-@UqX6}autSzs>xa9elIOo@DnF(2$bA=E6uyIiq$;0y&i+S75-)t81JKGlJ@jCx<4Kz@V6n->1{Kf=7;Be}uQX+dxVdOVc07N~O zcViH!p{1?LbD_*v++y-k1MAlN_Gd$BS_eR)-sr(%`cfGnO5b+6gqZNA`$uGS2xs%= ziHCFc1CESGy^=v?!=niQRqz8jLsfIhw?~fEeT)w_6!q^HYxZ=I0a(--`PW^M4_jbR zYD%{5oJ5rU>iy53^=U>$UUCZF3V?tc!a0Dz19`74=q>xO_JKC>!OKLI8%a-WBRa50 z2G$Eb2Tu^W_+53vkhJ=hhKcW0!or{2qOd6`pba80-N=w@a{jM&*c;@&T-!IlXrPoJ_X0O)%&s|JAroe{xc)_c>?`7&%9qWs&#wzI_F4Fy~Oxz6=T(wtnfYI0EoIO zgULX?eET2nI;5AFUi&o0M9TJEv*IT^4fpmh>rflIIzT`13r4dD_I}nE2XCGEB2w{o zzVDA^)S0^g0y{m3_|vJ=uGSdUdnt79Qcw6|^;gop{*SW4p2PogaO20{<$GoOBkA; zKmQJ7xMIi4Q)hm6y{;%o@Nn;JS{!7)0MKK3dR{BAoiR4$ z2+x@ebkXNRz63@bGk}-JQpUD6xjgir_jF&(!Q<}_Js$e5wHW}h{{jVv2t^mA6_@vq z?WSsT^@(c4(1=xZew`UHvtoZiUK`js6KG@=to2P$xq!Lmy7K1IckQ%+_- z*k!Ho{KvA61*l!T$l%Z9V+-DWr!}?L=SO6o`x*E_OZZy>25>8P^mVdM0 zG}2xcK9E25{{S1u3cu#;0f4+uc${sPc_7r=`^RU9Y*{~L8Oy||>`RvHYsyZWrNkJ+ zOlHQIg=~!_x+Pgo`cjrGNknN#_N`Kg(PC+_i$u2&Nv_}cUUl!U-*5hyf6nW?pXWTE z=e*DJE$BKa#dx1Z^F6+hZlRZvk#`=6hwdeO$Qr$?rKzc>qk~i@qVWVGnhev|32A6H z3r=)^ACMW$Bu&^dtcp`<`cV8q1&1FI&H|U|&mA511;iaTjMhnQ5h0pDrmfTABNh%0uyO{S=i7A8Up1OfkR!NS^6RGMhE2o~9f3F;19y6l|wBXtXYGsMLqnNimBpXsJ_V!P8nQ9x?Y;H#TQpso< zP8|cY|HPj+8`NVvop{cf&%)ahDH6#CCg8Uq{*J6d&I{d{ZR+dlTjW#j8Y!TvK^}*q ztd1u8heT6|uqq~}@VSCM)#lUkOu}B{h~iN>y8C_dM+*XfKX#rsYe=6KHsn8k&N^;X|4F-K%?9JB*H)KEFQ?Q>W5)qn{zd&0~i}+0iL%^a* zY!R+@Oy?~6nMK2RC8|{s({fbUn+3|b#i2>b(|K;abNi_4TW<6Fyii%Ky0+wy`-HzV~sSB!@C1wzDUjwWwJQz zvuuh~zZ`G1zSuGap75GNYy0_INXLqFDjdG=T|VQQX0gj^07&r}IN@kCvYJ1Z_TO4A zyf>k6`a5<{}RxvyYPgC>rWy5yIU>Iq4SpUj^wMeplbAZbEsu&Q= zw;f6XE%}8ZkItv|hdpz}wV}_iFZ+Y5e1%qNnDFM>X*K>o#saD+H>BuFS+7t0Cckmr zsAK}3#tu;C9q3`Q=ndBsGAT*aXQ4iYndX2Perc#^RwKNccrSeQzTxeKx*~ojDsu@` z@ZW|Ohp6APlN<_Ph;%0OrHI*d-|fu>!2;$`Ra#rXy)O7Emp4S)%a?Lin*@_Jz#{<% zDCnjv^@_?{0mQDp1_!Xa@7(mo`@IejQFf9GMV(i}3u?tAGJsD|79zEe=l9{X4)|P{KT2 z@}k#-Fs121+46k$wL0rhkFmFZONW9b2GKV1fn8`CNz)+i@GY-jT{6107&TFj1ucSB zkayncYyp*&KXax{co6|g8XBM9>;e)(U5o>(xe+B-2b;;GMT4#THTm<+RVRN_r{aPr zG(u1STmRHP(N7Zj9f(%fC3@_~^6>z#t6%>w9s17|g*O&d#Z{TvYUY)S_t_(Yo!xn} z{=ETUgfpQVQbgH&Nc4-V!li>5_AOe%Kk5lUSi}i3!SYF46duaJ#3c8L)trp;Pi$%d z6cH+r5jBVMl0G}EIC%aC`K_x>CrIYe)-(6hK(MGZ#9+L(T&XVItufF1Zr7V(7xqPZ z1muZIdl{m-W84}eEqMpDI6?r{C<)sk_P?`N$H9=6QA(KaiY?+1EYF&6jeAmRmTP}O zBtODNnJ4a*WZ5Y*s&?*-jk7A*U3#FG5(cEi;viKGuC|d24^I?)-r+v3_n`!AWZMuYNXh4D*-dI>$(7Z=IaTiO}*o&h1ZL{UZ~CrEJJZ?D!B`U7;Tf zGi$$){h>1Q#Zfy2?Ozl*jEzh>ZofT8Pw%RL6z&d*%G7X4^)3Lt9R<*55xlBKF87!& zLH^i!GQ#$h)ch+jwx_s$_eJ%E0Ga#DMt$7H(0crU+6`)-<43enF+w|N>Ymr{Iy(TTUQq-xRxCfdY#Nu7+P5MZc=rsFXt?+6Uk&hs^O@~czB>mr~G zDaCL;LEH8fD{**|UW?O1AIek_+G4W#(E|Q!_R&%2JVyvqbxVi`HjgShJze8Z*8U5i zlCcM<&&Y`i+V|%_V^PrL4a=JF59Q$LH@PHtynU%+ z_n!On)sO4Ig`J0?qcM8Mj&Cyl&|G<19dg$#R%GjnGWfEy0rbfqg0`m-lZfKIox=>y zhi=pB8QSCp4AWS_0S<8mA9EaL>Y*ZIajQem5@-Xp&rF$FC^{Sx?oT!1PC=#56pt7v zmIW(j*2i)H#a(_zMZDu5d%5Q_yz{xe9^S!XCkw)g{yqjVC>&29pxG{q)`E>rDyRp& zi%GR}WDcG^HL5CDMZ5yCcj-e#e_ExcweXngG58WXg$H&0eq4?L)4TkjgLx_~y2`K2 z>Lq6{dc@w^LA{3>V{6Jgj%~5iH6qH>sY)f4Fth^#G&QxLx$+%qKFGT ziW43!)!ryY<9PPMb~7BTw17MK?X#Q|90qgKf7tY%m)#}gW-Zxfg~6%$VF@%eyP|*6 zE`A`Q0xKd(epQG+)DC+wLTSv;d!pfxue8owYeCVi`j0~7ORw$@b3xu!;H+{tDJ-}bZ$Map*wSuKi#2K3M>rdyq%n8>0)PT*P6s$j$?V>Gm zr-<%Jh<^R=J}5+1fM&7-)b)$ld?(w=^7~b;nbAbnCy<<5aZ`pc0|j8Ks)nFoDI4qO z^tQE{9w})$z--9#nT&RKiL%oG4XS#KZdBRTq_6I|39IL6FHSpNjbKRn{1OkrQ}IC? zZ%762+54O}3=}FqmL)cp+IKo(pX^VH&K^D4hL0xsvE!_pTvAJ1KqiK>=CdQ02~X_o zSnvNI5C%sxd__B`T*E%| zD>H<`9NnMFG>R|XpDjC6Om_4xn)ce8Zc-h^G8yl*QCkFBnqmNh@>?S<=Ux`o zc1$1^Mf7)1G^%((xv~Il@9LBG~tn_jYj;O7HC*1?N@)8{NcTDX6lF4 zP_x&-?tpAu_oQne^P$!Z+?2gIuf;q%ZylnqdhL9hr-g!)#DtsVPZ9Q%K_#KM`(^IrQ-x=$O_CwRK zYE&$R{c5r2TmK+-z=(Zo2@$nHiAg=V$jc!Bs-8FH_{9@~(3`c!z|P9n9ky#TFgrQT zo%Y3CH8!vE#?nuaiVoQ5qxrd{snlPb=r9#R9zbYaM7Y{Q4~rdMT{+bQDmu6i|iyMJEii?%pJO;`_;d{bLZUWob#OLoJ&DoUPVbsRMr7ahLg!e zv>kBEV=KJBC@ipzFF{^RI+}vjheU7{7HuMEJ>VWm$Py1zEBindQ%;AK@2V@74uO zoD!>S2`g$=74U-gigcd?ALN<1aB<6S;b;7PNA4R1h4jln`vY`eYr+%px>ZZ^nrn{L z?>`5;e59|vfc8p2f4(Poe;gm23I8>{QKY1c7praj2HGpr{rH$BsX>T44&IG&%Dizb zX-53$4QQ`I_a&nInKm7QW|>04vpL^dzh7Rvm;voo>E2<}2kC1L!Qp`$^De8lRAs!l z_(}@e)AXmA;+)nOdlVu!|KTR3DY)3{28Tmw1xEC58}Mx@9t*iA`Y_jg_HIt^v+1$@ z&>o0>o4>g{#zZsL)_5^LYVmSrM(&pb&>o0>Ac8&LlAB%I+(Rb%e)-&xOrg_NXb(ic zTMkw8yeTDKt?}0nd7O7^T91u7v~zMK z{!*QccQ1qk6^X5FOGXQ)pgldB-&BZ%IH$V?Vda@R7WdQpj~;per4<=|bc$OnRmXpx z-O^;=?DhYoYziD2pgj;x&!0-$Ol(o*6#cn1<+UnRg(cS%pgj=H%nq@87mSR|bmu44 zH(YWLe$KLI3EBhEq}0z)=FT8kJCy?7X>s5v=d$cXpgj=HVaXlD_@et?IscP+Afs)z zVC&)O0`2KqF=XSWqoK#~Q$vZYczaFP*`Vi8`T!$3KC}Aok3P$grNGyc1|r(EbW=LM zLVMu(hQ<9=d7apjtCk~!@u_jw)7=ckp*;|trY-(x)18`cc-w@-&rGs>Q#y5T53~oO zlVhl%muwfaaJrI^5p`klO7XCM6SN1S<1^XZ(8%)3UW|RFI^Q{H?6t+$Bxn!Jwe=6A zWqy2}x*-=!eFsi#AD9|2Q>2m1x0u_or!2t&Dk;e7o>OjavL;Xb+6mi$?i=thNYY{_V58Uw_PY zs%gb>LwjJfv<_*VeYC4}u31DVIiZ=NclQM;kEEZY`)qVz{jFwKw_KrZ ztk9mOClwFE$+*GRsV7ap)RaqWDPItfd1^WHcnt^=!d#c;=0wQRw>@p@s=p@Qf%ZW3 z)rCA+N2X!n+?Qkao%+3BwwrwyERbcC2pU%(5U$H9^_~#Gomm5Q-%M%7d$dKKD0PA*nXyVckVrC4@AFt*s6g0 zm6}!7;IrCW!6mifdIC1ko~ES~ckj6_JZGgfG~2R8LU`lMlMu+ftip)qF~n1Z-gMRW zGj&7k^KWb{aT^b`L3<#Y=UE?Hy1}{MhMk($@XF7uo~SY!*Z9RIt)W*S#>E#&@uIQ}M%G zQ)mxFGyZ0ggIXGIL)zAS+pVTLWhX=z6`?&4AH-PiQggM+QLaa8RTGLu1YCGg5EZ8i zqXO&lo!&kNdbt7%K5rN=H{?^QbZ(FNNF^<6^ zd>pWD-RQ0(&Nxm54`Qn>zx||4msC$Ra#P76P#Dfe*fy~gTyk3tuY z3#MXjt3FY>2a$P}^GN1;9^)XE&wU)I-u&+u)&$m8NclnO#^j>$l2G&xDJ#je;Y6m-7EhygLdL!xY5&2fBVBh1G zA5eO^!Wd8=xDb(Iu64llaF$~J^$o2NCJ<){m6ZzPjYtWv6+#Oil=(m=AOw-fzMuk8 z@2IR)7{*fn6_2w0_|{D-NU7d08@)a{4KkX`6$W>v-Y=qe=WJDH;>90glE#WJ@DL}z z$8_;D8fn1NNZ2EN?e5v>C+?izbt)XJ->Q+1`f$iaB7#Zn?2B2?fxx|@2cf6loVp~ww4aMfXWlLCw zSOg4%CXoTS_Q!pQ~zV=Gx&gSNv729k-QpMh4Ryy(kElLtmlNS3h+`^18&0}_WM!pTS&ffiPJ z6CR2Gyw%PPy_*bH4tlnWUD_5l*9mR{4%Z!*pQF54rAW5$GW_7TsprmUSn;yO+uPys zPB1qTM%JE0V(HxxwoaqfY-)Ra>cda=)X>xI zx?AHV=9nMT+dnsE>}%vG3d${$16u+C1H3oMjNIDo9l{;qRp)9lBb2`MV51Ef673z) zSOT614FCI-{;_o&m&eE4KRP~NN)T5#8D_m&y8u17qkF1*GrP#(&zdIx+j-uT)p}h3 zKS030;~;8>CL!&ChuVKhl6hvV~Pd^fR8*>3EdW8!tDL4cMcxKj@fyb5Vb`nEv zuuIqAS0&$VWUUrhwbC%4Airbwt`b!DzR8bZz2m zqyh6{QRO1>vilwX`HtJ~lyWEdKU zK)TUe*tH!~DU5HuydRgWWR73Zm*#%v0nq~3>kJi!?`$7F$Fi#qJ-xqr*yg=i46`K? z-C+c@y%Vt35nH(Sj#9X!O-Y@Vekk8iZF&8FIB*mi|D6>UW$mx-Jojp8Pqv>#QQR)D zM3eD&3~*J(nInxUQ65;V7uqZ^=j3s%)@U9qF-SNL7?&Wm+^z@b<zT7XOCq4@e3RWWjW5y&Tk(&MJ*zKHq z-)=E}r_NxDC#O`tO`DF?npdyF94>?lAuHebx(imf{aras2>GB*~tL zwgcjvtI$zo;4Yl^%c|?FY%s}5*`jI_*wXhE>cRp7Vl<)bHd+>T&>{5tI>d?Ll1#&> zo2&KTIe#;@=}y#@u_(##wGpb;vtL_f4NX_;l>wiso7~IcLzXa%s6#HKo!#pR50+Ym z>2A||UUBX{uZe28{fYdwZ{O8%i3YB6E3-HvrA)}sPt+&zaK%JSj@-GY>=(fUj)IfO zFkl4fJX>7y*QS3gq@2B`I-RASJoR3!>F|TVNvrD}PO+>>+cD~)g6||=iW-6; z8bd}B!AVqchvT=2f2VBrhA6 z^?0Y6l|E`eICqRkJFTKJ8{{19>8dj$I8=&Is&r;*y&~To3w6D3!)0KFq-Gg7aXI0j zft}rdN^4N!0KT-iDJV+#{>Y!&&aJuNMnr1uN6)opm96+LQ^C-J?@C2WQdjOR?jnL6 z1x;rSFCz<22j%VCmtD!0%pL3Z!Rxy+h&jSZj*JKZb9#nz@mdv*JAd%V#H&1S-PkG1 z-(Zf!0KeW|^wv0Z880SBBEwygz%#`@?M^nfvpT9H zaMvP;&%4y`Lqin~f0!7FHq$AL2-r%8Sf3i6 zN5m9{qQo`_uaP@;`~X;!KuL3*2dl_)_NZ?*z2lainDbwL?R!Cv=%_s=dzr2q7 zG5SEWLvT-cNRP7=zo@$hGhWG9 zw-7!pEb9paGxw{fv78%xkDFg4wi^G#sU^nWALzUhk(tBE;b6qqwpS5P@NHLU*)4|-hQLQq-_Y5NY0 zda48Fo!cneL{@;Y!_SPfGbXQjnqDahJcDWhq|9%ajpm5L36%1G-f7ngO#*Utu7I%< z84gU-n>D<;nsp?xj0-%e+#7QBZCv*9GsERqyV|elKy24V5Q0nDT={#4WV38|!P14U zV$5y#T~I{9Uwc(13&@(f7N$t^fE12)P$nV)d--u{Uhs&Zuti6>#V*#L##6|QPLQQ7 zc&5BAd;HFQZvODxSXmtW;Dm=h^DB)#`Y;PKC=FymYHifuo0u3bWGFP~7aaI-c}>E)$m%iWIJva`v?kVRDQ#Z_!k2eIjOGCf-+n z#wGM(zl(NY1;{y5sMQ6i#QDhPHOO~l^VdqQn)}1~Mz*{i-*$sI5eY~9T}#fkKlxSq zvt>KF^BdR3?bdrOZEV7XS5iiYuyfNf{Q>ixQLh^ADP1hG;8?H7B<+#Zle&%7=AIxf!t*t<_B2nSvyN23hcc&F3s> z0;ctv#EoFqpq<(#2yLzTGg^;NSq0}0Jl^$d;e72X&>1|w|B~p4}ES<6<2H+4q0XCYYqt7T1yEr z_n3MOHiPJync7_mS0de+F_JOIF7i^YY~pPnN>N~4uQBgSAtPF4!4kMsv0M}(OY&jd zx9yn!@^q2ad*{%}0Fa`UCNLyz8>8!%r*{g9THd#~vOVvMJQ4TJ=hw)=)p}*r)FXHM z$?UgkN!^~>sruO4=|V~F7BF_CdNKv5z_w2sUKoxxVm&e&y+z>w`*g{}H8M;V$rvjh zjW*u;BoXAJwhhUnE||WQOKu8Gkh;W{C#{aY#y#r~v!9x5H1sCM%VB|h^Q){YxZV#M zGeeAhG?+{E!^Zenz>0SU9NVy(|J_0RB`q_4D{z*gYjZT7>^T%3v!`o*wO%gtXLO=F z9qsFO@?J2Il#^MT)V@(uYL%tCWBGi2Stf4>@bF7V+7i9qg!^6)+2YB!n!G zxVV0v%J~zlElD>z*A&_^Lq=ZOfrytIbV(`B-2dxkg!+!2(q`><%#ch0vcCTukJ5(Y zF-IQvRAv77$ld(4P|b-+643+_b!>qSP}oZbU^2~{%EW$MdGr{iE$nFuJ`9j3c33=u z!YCcQS~Yr&o`p3ROYIOdoilK3j2s2;J|q|5cr6yM7AMlU-+kh_+e!JJ+*kJn-Vngd z9czcjpzT+pYJbP<`i6ts-m-fsUR8HDRlbt>b3e#0@9aNGid)lJ&v(74Kz*&Yd)V+L zKTd;*5&x#zPqnxWnJ_qb1r0V3id^>eZ#E;J-$%ge>r%MS7~IsV!l~O@4cA}s2-+?4-{~mP^I{~RYEt}E0vUTk)VIEHja2c!emQgz>htTev(xeQoJ~Sa zXqA2GI&;{6HE>A+TPb$gtlq8(6MNE!in%+vmST5`b!OuYrdAj{b>9MCg1%^lQv%ry zIZ?kuIpXXNWthB@Vf(kmO2#N0cvyErFsYSY>AZf}zEaYg_FW?&?g9sHZ2D$OZV6W# z)|kNuSeA~ove>=%7y>gS5sh;H`_u}!fMR3wnr80;7OtMLLkCr^a$NbBFHaZ$FPm7E zuCA-R!E%zFosu4U?axMVAx%B~z^F+B@5;!>`-10{_r+{U=ZozN6ImEitC`ygzUFmx zq*Xs~oCBjksp>Aibwm~QQ7^Kh9UIhX$!#PHzQT1SqRF5tx+3V|=^NesdG8G$#37&8yYH3H?H}YS1hzcHFbXDTh8S4gZG4^;OwTBcko`{A6Dft^{1kKT?d;QkG0^MvLG`0IO~n>|iYi&hnQi#fTMN3Ua>Wy!sCAz! z&~Is0kwg3@TOusgG5`H>^&bFP++z3FCOgYo zeo9Wu$rh|QgnG1Ag7M2}1vOwQ5kgX`6yCht2*-$xyFKmMOf z>uETmgwX+7L4<_^rHf?P+Ao};$rD8yh8rK8INtcD_n`X zb+EymQmdb>L9*-Y&Ch6E>s@8(#XQX>B7VYc!|RPC6WF<09Sm~Vuf+e1Djcw@jZo8}6**-6U|5Ie z2^RA@;e=(e%$VAvV6Hk;98(%oNZEVZ__Xk{+#<2M^-g;G$(ATkf5Xz>Y7oh zDiWZ4VK61}WwjL2VJA@w<-cR+g)xawanAR=o$umBn8_;Cdw|;{j&3)fd9wcaY4c&n zvuPJy$l7l zVbzo+>ma#};ZBmBYjT0(Pqwkoa7IVNa>a150H^)(lE41yGjJTs-8_o4CN3*MN8~?@ zgrY)`nY`ol=6)K&-i9oSv5{BlcjAYtJoQdEXRe88KwZdW$Awh&$?YUbr}@8gp?rTB zT%OHC3FL&H4tW@a4<1{X$!G%S`LqfN(1VYUY5>2%*7xpafq%z522YQnaALM*R9qR* zlgKx-OZUSrVT57m;al6yM%7i+RGMll*s=9ek^Qc0MEqp}*#OJwHM-B)Z0m6QpBr3u zwtfS^AGUbTwz2|voO8)fF3nBND@iQL%+J#gO3m@hFG(%dHPkcHGXcVqiV^_a=nKlT zwgz~d&0Fhp+qe<`Ed3R*ZsJff<=Af8Tu+u~oSbt_FNtpwr)isJIFtxktSM6A!?v!; z|K5E7AV5-%bJKQu9~=>n#bUAh+Xryp#pyh$A}I#5Do)Bc-Cqv+-{hsOth{KLmdYY6T$ zTPzCPUm{a80NR3*c2lin6wEfIEKDctBY69#)8N^~U&Xw-mBv z0EcGb>nx5$7)5ul!~goi-n=i`dnaBx_5!~_R`3_}_;wy9NjCQmeVURjyw-%_kVlVC z6oIKQ0;gu22#qZ6Uc3+}(1@f6JwcdDqWP3ga&H$b+c#TOBVk@|C08rJ zK~HxzI3paPf&2}pd;nix`n@Qlg$iUBz$K@?Tf{Q&z#;?F{l=EW*bqgwaV7}pfElNz z9caUitF6m0Pd4K0{P}<;VIv}Tnt21lPC{>yA$qKG|xLEzg6^KT_Vv3)6cz?21QO z?)Zy$(aZtTT*Z!siR7Qb;XqH8g6~-5*$ViUkjwBeZz#eAMmLbofEmXNioKTLbLt=* zKF+V&0S^Z4GN2?<2do{DDyhyrL}o{)0Rv5E%#9>f#KR0wM2MM{r)KIn)fc7#HO3d^ zW-ZfjMFYnt_%-B!atIN@f$@q9038lMX#rs(({}h-$DSMs11wz@;`7uwPb)AzEh&Tq zL2m?r7rIu8F!O8xek*!Vrmcx6!Eo8xPwIjN&^1JQ3dpIjzYCn zz^6edZ(joq1O_Ic+9545!SNyD1_iXWM0Xv5T7wVtV5E*d?)>SWgVWc4L)U0d8QbZ! z8@UGct>R)8mhwtYen(Jt_o;+4%0QKpFcAd ziirC?mtWA93V;k#YSd2{$HDGhs%J|I+$Zl43ncou=zM@PY9oIgXl<#pk24Cf<5ZpM zKwBV9!eR}RvGu^GIH5@b!bzobiP34?*H?_I3u+V))Z1%Bpq3%ww~IsZM1^WGr8FW2 z1hAk}#Cv-xgkwb{wEYl&+*i8Vb5y=Dz}H%+WI&Q{C;9bO9Dgp;YNbRIkS>tnkgKkZ zr_)K-wHwAQh!oR&TrRaO68IySAg?}T5I2ych%5XTTOocZ5j>Cr97$a63-Qh={P>I} z46lW9OOU8U<|yNIj0GN_a4His$vOceLWVv-y=|W4HwgoH*&48rF*E1&VK$ux5oq`a zVfruoNx$4}hlDA*>Q^TnMo{F&8gkP}@yt8sBgMcQA4ObP&BjTlpgxu-ABpYpyG zc>MC(9O!LeH)l2QV|=2D)LRh`rDgbzQ}Ssn9GmH{%G;8x z6Vfr*OGrR4C7}@aw~F}Q!ejo{VP42~JmeD91egaRvrEhxacYO_3Wy-sWm)srUb2I~ zrVB!sxkb(_#ug?5R%gkz z=g)msf{s*p>Q1ohygg#OPI0Nq(%uOMfwpPs99e4x3(F{@stH*6wJ+)f+$s!Fo==~s zC~;azUECoa9KKrCuCCP5ELGJFP_%sGb)RgxC~>TNn_vqi_+avjDYwCYHWQlypDs&Mbk+4JDXvzHgAZ2G#gX5k7IP>&9V zrmTF#73P{P+ag@K(Q?`>APe28Cn>0@{R_7$TtRVbt4A7G_O+rRQ|6<9VlQ8`ZTs^!_Z<7*M&vV<3#Ryv^VHg7i`*fZ2)=4e*##@nmRS2dZ%pw8FvQ)x#ozXa$pbE(POEC(!TMkGmmjM~yAnY>4Ay zU_|WcZ9Op;jqaQdXD!A}*;7_y(}p9&t=nxU-HyNCEr^QhLo?Ect-|LM9NUt6mJS z)f8ZzZi{CUU-U4(MF?>FJm5b9Y-W(}+%SCu#3`3|5A44S@HNdoor0Ls^Qta~1X(e= z`(5LcO$bBr5D8x0lh#=my*QvX#{?_(DPRU|f)=}&X?a1DgWLC}*0pK-*3?MpOIHY` z!qi5#d573JdhL98q(UU{k6tsrG2g{9UbHX4G?aEk>EwVqBP~3nF0h=oo~rfR5N-pw zfp1lS2Y@~~5s!xe)98)Vts%qq@9AbAHvvtLCw<#h9o&Aeo}t{^0}}1MshdqKY+eQr% zwUunkB||taQ|rf!EUptG{eTosQU5XpKRpC#EWyArX+Hd|qudrAKoxT{1)4@>3WjYSk8G}=6qL8VJ zz^9sk-pIF1&+F+5jJ}b8x-qNCsHf9kqryjN6S6FA>X7S#EJd5dKPQm`#uAcs7_Jx% zO<9P?HBbqrEz2xTyaqBg#%nauqhX~os}kG+8qs5ju|S6pRbfJD!5k6qq5qIGyBg`} z8UkiE7g0#sT4at(bg%Fg()j3ZYS0AM7;ZZmdiL+x{||u5{rXfocuz%zhv4Aw} zAly#JcpHMNRs13Y5O{xIyb3onDT-_bQ^F0$-rdyIQ#X3W`+{B>&dXcE;@a2Y=Y600 znsLwLGz9T+2&`zTSB7{|+Sj#EeuAPpbuC+$nAE$O`dR{sQk#gtA7i>qV-xScfc#-Twwc*f=1)`VM%Uy*>R`+eWrO+keG`o)DV=G2L$aHcog; z!6cj#%(J1{Hk+d(Y#D1~OI}Ilqp$q$?|#guG%_abzCGK3MtAPa+_~R#HGC5%quFvC ziH&JdEOw_G&2MzeA{>3emS&t3VjLD>kVT^?zKO?+NpZW9nr#?eGv4tv4j<=`~j4M7$L{dtLPw;XjD2(K4IO zyUlN+WE@WrUxEnF`i!r4F#IJrx;%e>aCUUsv%7N70D!~gtk4LW-#|;8MB;qV`#JdW z@;Dft{Mr-Wb^i9fel<8ad)Euz_YRJF1AhINPO~|hhM6FoQI@4yUSywm3WGtP!9K%+ zt)&rcp3jgR@#NlJ-6&0R070Bv2s4T@oMb!^rzbxW-k*De;Yt52IJ^8H_TP%mUFV0+ z-#UM(L+~ZKx2g09n*XYuN4N-!sm4?b6(^HaFJ_h|*Sy?WdK=`6C>jT|cpew-ntfnJ zJda!`O}Ys%Q<4VJBERvHjev;d7Fl|CZ(!){GLNz#3Fpza{CyGTIdkMH+U)(%=`>$$ ziOW19HtL^W1V8nDIX@TvcNp4K8#q zW^8Bmilu1Pw`n$Z$Xj_!A{b{V+zJl(dpS}}Jd1(qTZ=5bo`=CS3dd2_T>%KYR)Q@7 zyV?V~rO-<@Leny$j@T{!EeZ-T$>1@#T23ZW*1AdKF-)e&wuO8h%?kWYKPU5Oezh$a zkb4E~=KnSu@;&|lrjy;o7I)~vd!8P5QC$I2;(!S?VB1^NYhSkR?8DF9S(Jz!-LgpY zq}^;Zb$8m_@}hOXTB2v!XIj`iB>% zP28?H1x+U6bX3d~ZCP_p!(==Ia*`;{$BL4pSsW$Bf+<-;DG7rU$#3IgG!-qt&=$~T z6iS(b8GJYj&id!Cfkbk;+};cNKVjDr=%6>eI2c?EKqF%pXXPMn-8wW<`Fn=BJp1Xa z|M3ivL+1w|la?aMaJN}pM2ae`b^h~w2S0Gfy?n>-f^a-;aqG6wQ$cocZ1l^Q-6|uv zOG}os+1D+MXF}_$9kjB<2dx4K323-Dp9WIoQT|)2R0ICoe$1U9#9#_WQJx2+j^^-N zcpc@WGHAlc(aCbI-x%80jtd~UB@ zXOU<*IG()~XP2j^Bp;+&?6*7na~SmPTXuzp?}DTLS&u;*+Dc|o4inImQ9W^IcKIEO zM}yY$R(NzQyp%H|7&#-cxruKC{}12n?BfXd-W@3l^ZS4`2C6(g6!%G@JDvdPO=|$T zj)q0UESje%y4T-SnBCX)Ols!qkDwIlYUW?~rK($TNX+g(!g$ZaF z9OVx%psVJLGe90xx+r4M1ufgZx(Q+GZo_ODX=8B30tMUp3`A%&U$i)k0Ox`o-VFc^ zvo>#Z)fv8yivX}s;_KilO=m3{j)rJk3cW)U_YZ?Ej2guBu<$~MIJs_ZDu0Jy%^>S| z+y0Tk*CEqr6|>!9!Sti&aEdi#6T1RQJ_?nm1-o_!uYBE zgt1o)iecBPF=bfxHwBW=DLW>1(0lb zF~9`I$=S(8R|?KE9qN*t=qG@_gRbM{qIDgS0Jb*HrR=;G8*QSoaWN;Y&E*2@*7N-J zbCd)tm^K4+)kcyBkVgd=krppvTgR#(bA$7bM{L~W?15xOIGXYpLTi%8fT(|v5U%p4xnYQHP*Y2v_4#T-=LvLi-B(FME-Tk+mVXzfYg-9?cd8y8x^xJiyj80ukZwoH}|B0 z4-d`{F5b%(Q}>bsXq>nw;ypP#?kl|59k=JvZpQ~Frx06V-khbRX+qz!+p{k^5Q=)U z#_(!QJ#yOr7z{xukAl;a4<{GW82QFqPr(>`7bhQjesnlRDTr%Oq<;!HPy2@lC|WjA zK=Ve)O>5)b$wi>}T63M_fVg zeE2i#V+)z_#n!@gKIs1ggs9xrN`0zCK*{scnu(2*9ob@b0nHgdg)rLIC zv-k@hw?QbB0)jl9N5vHU)Qw#sK2Bq>4&ZbCH}%y(Z_xGFfBgpa*8bn$Ow+vBf3>^2 ziz$REoC6@zO#BNhPYKXK&`_^vc`x6Jzrb?Bf)_fPW-TPfi@(1hDM$L0-n^zaL@fO# zS-t^FqIn zZZ;}U*WKV*@ZA5y6R{TD;sF&xClYvMKm^+sx_@b0d|Mj#p)6`3y1@iotTz3P;&3 z9c%NEL$hIG%a)x(lOc^(x7V}uDxC40NQ1vS?f-ai8XQOnCf6IXY?R8GiuSJsXj379RNR^cj^1PVJKnB@Hea^l?hMNdZ zQ!i&&EVK9i0<6b}gMS2r-v31w$z<*ZFuq;a)}ZKLPqlq$zC+EmR| z0Le8#w5MIW3kia0CTv=#F~KhiTD>R3M;iMKKtYmqW~UL*O$3fdFVW(6H&3`%uQO5H+{aIy&n&7X z->(Fsvu7)kICLI{XJKU#McE1{rtA-n&^x{89bWW~Hrl4kvqct7;ycV%x+8CM=~Ygy z<9eBG&xwAi2&!QhJ~7)l$r~eLj?o;=qA&rAqH{Zs9gO;q6Pv!%L??ymMTqBwl|w^y zW@wHc6|x%MeByDU(lmC|p~z4=cf`ZDNft%s7*Ucy&K2HQh@ri~^>)k|-rhqi(NeQ+ zYt=S_xwS^y7f~?ANnwMI3k<+Fbe#iSZP^Qu>+*Ik`vWzOghJtq1KGSHxn;FayBhBK zydvd%mr_nOSyS+N!NydfAVS?nr3y~2CwodmGb?yAtG-;Up4V#mR+-m&F+o|j54HcG zus$M^w_N{FYYAjAMUjO`K8dni9-n*+!A9bNc}U`h8O;w&0i=Xl}elh5FoXi z-U^7ODS&W%NJ(TXD#N;&bkb7fXMU3pQW0u~GVMILs)o|dQmrakTUiF;pP*N*Y1VM- z9xU?6p>21(rZB{rsEydxv8wgKBBUZl#VXWk*@i?w*!V5o>c}|@>BbAvP?$iWhKdGR%{Y&=%R8F z^tPxd^tCWhiucFM<6}hOluoaW;_z~K(f=SFNF7?Q*B9#rk1x*-Ij8!ht{(>{7rn3S zdwG7^KRA-dPHR*8UGG9=X;tAKu{`h;mFyN_Ry?g3a4f_EO4^HL$g7e&Hl7=@nX?EaF4At4QcHE83=Uigk|3D8e&1OpXdzdo-r*8gN(TMc21IN-LP7)`$8E zM%v3sJOl4w3Q;aDojo6vYDwO-aE+J@oZC)Zu+|Kna$8)^wm zVztcJTm@Vj+IBhI!EgPRtX(tPty!FW!A$z5nG;$sE?|UqxqNCu5NN_5s>!i$%Kk7? zv^g5PYSVT{4-9Bb_K+rF30eN9MMGuB=8ea%so5#=H^G}~S?ecCF(ri6%*!Kdi=xO) z_^<0^~7FXR(YlqwUHbL%eD||FQ^=;trJ|i23Pqu4}MRIv(21l zE}})vc6roOjoPC^@l-pOer@=upZOrH@?L{ug?5w4p2vJsnpnMa+NiQ@xz>|Nl`WG$ z%X-jfJhKqM7iliFullqnixO&vrpXvhp1_jS$=GV3@^hypx*PGc`XFp#g(jHG!n1%p znnk06Qd{Y=kieEt#EH*Qy-c*QMYdH4;7>QCr&U)r$U&`I=8r9w(Ps4#b&wYEg0g@#KP!0S@~| zJ>*~l;9M*isdm?ib+uBnLh9C*`>w6>q-r8%pBGEdPBz89k~K_?g`ON5}5 zGRnyiXq-bcu+&%|y1Ox{o6oghm1U?RSKVq4y;cpcTeKH12FpaKFc$Mi;R3ATB8ydF zp2e6>uxruD^Sa!HJR{o?WTyB}xr2^N=ysN3#Wb z9q3)oQ;>~&42eKk%^4;zDivVp!rU2tn@ZkQ6peEjIW3|VeT_8LwYja{r>$<_Su8xHTrY2sB8^Gn3(89v8UF-mOr49+SP=I1_8yC}nieQw&vA+LvizHNi zlBv+fvLgah;W$hWp24w*5+oM=4*z`8jWCzb$a0cc9#k7tzw}W0l$drHIDCSWc8WQ! zz|6Sw1SnfN59?f zcB4H7R+X7@pHn+D?;D^vK$k#@sFP>Xe%4mp?_JjMLg z^WCld6%ggyZIlLGzg06-!-I)FAAr6uzh!#x>@oLRrAB$>wF_Ro+{c-(c0V>1U1-kR zHG_25+rTwh3l;UWB=$ES?q5a-Y0dg`L4m&1$kgwSf%weeco1_euZJACqV(a z{9vBrP>rLp=Fo0VoI=xv{F>YZvNVLQ%=W6W#%R|_TGWI>-m^o|tA_|5`-l@*4@hUF>P zS4?k}z?-8ZvJt39xlh$hl)MnzUf4E5qIOIU^;aD{rfgeds7?oB^k}usM+mNdF3&|9 zijq+&<_qjpF)XMF)ym2)&!+4|-pXzf%@?q9YG^k0GNzqi8AD5g@3LfursiL%Wj~)# z2?TF4O_3PH!c}Gz#e0qG6h8^$q=@7g23vC{*kqV}XiO+|QT{%wm01=9#0;Yd%HAMF zf+FSpau%tY&iQqfTO}6Dld3Q*)33rFa^r~!O8ymoF)G3trZw(m!R@#zbGtfe zfD?Y^k;=B&z2~`1>xU!EzhglBJjc5l+|z91O4qGs5*4E<22q{>N*Ah6OaP@<>~ov= zZWyoUB)>InHaJb@#%??uJnih;)%0ckKy=yy*ae z`Cr*T;F)xEs@SVl<$9>#ORDqICYKT^wAIKdc!i#uJ1O{OfNz308N&~zwly7<7N3yf$zadm-Qr2BQf z;|S5!J=CpY9Rk}ytJ#dQW&k7UCPQ}s!QG8TLVo(T^Z6^J&I(_Y+~G<|?(&T_ zh0XGGd?u=FpSo4k&Z6b2B)+*!yj3w$S10CU@kYpN`lNxAx7cBaPqF-*fbOSzasJ{p zg}w4j76*>ex_paiYIWD2xkt+~Uo`ql5o2UWyrjMsr~`!mzWW}acM%%DB2w|SV@EA$ z0M-T+)NH+=TS84eWNa%Fi}udq67i$ZzwUMI3KAJ;TX*ewXo=3Gq#)&>|Cw-pUJ(y5 z@UIUQmF8Dt&RHiHf#af-J1qIU@&a$MjQ*`N`uMQ3j{xM8JI7nnO?E+u5T#zGWQAXN zBIV%|uUr!Bmy1NFH~69U04^n4_jaf`&09mbuBW&>tyIIkQzuBVQX*uXh1`{?4O#f^anEi3#t@y?H5W%zgXx6IC z;^hF9ajta@>`L;Hf}XrD~9!yG7{JG2Qh|n0LQ=R zcCS$W$!;tDq)P*BmtRSkcKshUu??*btSCZ-vtC!1oLpxhoOL2Sc~?642GO+K+*T(; zwiK06XnAumsLVlavZf;ZoU|$7w9_^v4YF?bMh7Rf32ZLiIqFa-yh=0m^@5|8eWx2f zCFM@hm!7qI`A!U~YVrvvO4aZt=l=v8E0fFh-K-QdP}B2inXdFcnaCpVI44&Fmz+mv z+_H2o=We+s@e1G8bk=(mglulbl1BArZR-M6iSYD|{0SPZw{F{QGe zz_%Rvk~6e1z)4{EOAZ^}NF}3Rywd>>OvLk_KTHF z8LDelPt%(>>QZakX5^m&LXG)ucoW8Wk%my0ox88mDdM*$Y>Qum;V);0!SU(k@IBiAfE~U+*b6QfV+($?O>>a4 zYz+p)^iRquH20GR;w*cfZ(%kN0C(5AAAAMZXot^z;m=oilS7uT#~I5?rm{9siK+?P zKH&`!c%i`OuIo0EkN^;|YbAw2nhi^#oIm+N!Y7%ZATrL_=xX+_^g8Vd2!Cw$m<%hv z_qd-E<$g}Vcf|X8Bok1kOcAjpy81|GE~elQjnd_8ET-X2BqpFJKr+Tla%-{hcVia*b}z;;D2F1=?q%&~nrw6lK>1drby2)cuCl+G7v0zQ`dI?hL#H++L1&?QMxA@$Dsc%K|+J4lETF~ul!lUQkI z|k=vSv@R%Lons=q9Am6sOR_sPV~K1f6DfO`P!i>gXX;$fbv-{k_aE?H?$`DC<#XNF zeSIeZauX33uCN_rZe17YmJ9ZnOQR-b(jso>X11y44A-K%_Hg1{6jtvnnf)OB_SI0Q z^F9{l-NM4x6^U=;;(DcpY7#*4ss{3=K7T+s7{dY}wb0d-&wZE3_YETxlzS^(hUsivNdT7$~Joa*> z)A6cH#~t^!LNMuK%(jM$k?9k=&d8y|(U-(`OjY93gvyJyT?2Fb$ETNHjFvb2YzcsK2|_p zB&;RiqX#KF^O=da)bg<2dwu-=mOe)cJ4&!Te%y0+UgxDEB|{emxxeJNJ;JQCPTr5a zr;G_xGML)U3U=I}XUCSkAI=D%mgLRK%Ubcfr2lS12|S+=*1!mhkI5@m(RJ0sy3+SW zAt@Sr^^?>kK6cRJ>`2}%RIP5b)8xr*{y4A4iUdFM){1(iL2@_hoMAr_we&)I*T0T2 zD{yT&@GNn6C=Q=EUDMLvpff|UD9sPrp#{(zHmjOW5!d5veb{Z8_ZP=#{6VMq&8 zMy5)UHgzbM^eSd1WqrG&rbYOZlB9uf`~+1On7{AAW6QKNOl5cj-DJ`_e_>bWsiI3R z-clN>3X{rqPb@)=)Yr!6C0m)BC|J<%z7`pQ@wm6Keto ztR|MFU513+W7fAva?=LB-Q;(rPaNniQK*W{_;~roI_0~Zv+2SHC7|cTNL8kHW4mfp z_`N&#?Om7TS1vY}*-u*aGKp16p*Vl^WgHP585MRk zG6?48@`uuC!ABj%Z!xXa*U$Fhl>*RCtQn|?^iNO6 zUap46gh&##wwi*22C0ywcq|+U)N?#1StbkNhV|Tif7Kkx3Y|X#mP_0M;^F!4u85R5 z?_Tp6bNci#?_0~se8@yn69L2+H<8^B7ceq0J1y2O{$G`v0&1XY=?yL*OY@E?%SwK> z3P0#5fc2D&p?$zX^+y0WU|)I^#+>XFn^jm`c{V9=W%31Z64BA{)hY&b+$Y2nph zI#^*N8Z^c2f{RbnVMtK5QtLeQbvQ?!i1|fF2gV-cmX*ox?RL7{zWyGaXW}Rd^61t8OKB#`n(>qMRN|7|37fU))__V1*sGfed=b{l z&~zwt^!X@Cl^jb^ka=S^0Qu^va6wI&b6f!TeDkv&szV%3arIfB63)NUTjByJW4H?5 zIW7bgJ+ywS{?vV>?8+tB=moh*G2TKSh;}>{0Xm=D1wxBtBP`E9LMo{Ot?dR#*Q9*D zD9HB|91f-%V}V61WBkq39JXPYCnC@0FT8n7_#`-K><@;R8mV>3r=N__3>ar9;lywepf)gPvJ!EyF^r=|zP^i1*#JCC6!r2%Co1T?Hxi7!Y zJdswbYMBap-Q;zd4I%BGBLGCWPwCL_sve&UJn*N520AmeMhMUcM-&(E=?Pl4#8xh3 zlPoZ(d$^IBQoU9V#X0N4&HS%Q3a7ky(vq>Kk}WcKUrbQ0v5vzUi^0fq)>=9`=zyq5 z96p?Ih4^DoPAe96v+!i?*3w4J*uS1kSSPhsf#z<$fX|e7cCp~zr(ZgiJoyrz0l}xt z+rY3QbKk9%p5xogBQ_y-m<@eu+_zom({|WoUVfZQ?*u}7f5+vWhU9J-kh9d#qCuZF+<@B z2!`}|V>AHTPDsDyXyZw#vXYzcQR{|dv&u|P7#qI-D_`|ct!)l|JMd@3h(1n40tu_N zK~X1s{3X;026Nowi(H2%u$@SG-!u1>ZuA6awp+QE=X!S1u66V;*KT|trmL&9H>7<> z55grdLSGkn;Z0ijZr$5L?(Wj5CGm2gfx<%BemZz$^pMP-u2l)d- z?}!P9yxdA)7=xt3<>Q;a;_@8wCj7uwbYLdzAG*=Lr(r`ML49Sti|7jtAZ^5f;j0OJ z$_3cdWN)P37>?IMlQAp+X~^rqfelpcJXa*+c@fTgD&!qfVV9Y{1>qG;4+4BPwiyC) s8|!*~4Xa@^J8%hCPWstmhwg#c9Q>8h7EoFYxtCroOBv5K!MKF~2i=v?SO5S3 literal 0 HcmV?d00001 diff --git a/test/lib.py b/test/lib.py index 723958ca1..ef08c2032 100644 --- a/test/lib.py +++ b/test/lib.py @@ -10,6 +10,7 @@ from array import array from cStringIO import StringIO +import glob import unittest import tempfile import shutil @@ -46,10 +47,46 @@ def wrapper(self): return wrapper +def with_packs(func): + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + #} END decorators #{ Routines +def fixture_path(relapath=''): + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + +def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + + def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" diff --git a/test/test_pack.py b/test/test_pack.py new file mode 100644 index 000000000..0b9e448cb --- /dev/null +++ b/test/test_pack.py @@ -0,0 +1,16 @@ +"""Test everything about packs reading and writing""" + +from lib import ( + TestBase, + with_rw_directory, + with_packs + ) + + +class TestPack(TestBase): + + @with_rw_directory + @with_packs + def test_reading(self, pack_dir): + # initialze a pack file for reading + pass From 099ec0dbd23bf46cba7618768e628ceeac7c2e17 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 15:17:29 +0200 Subject: [PATCH 0012/1790] index reading from V2 index files implemeneted and tested. Added LazyMixin type from git-python --- fun.py | 1 + pack.py | 190 +++++++++++++++++++++++++++++++++++++++++++ stream.py | 4 +- test/db/lib.py | 4 +- test/db/test_pack.py | 2 +- test/lib.py | 2 +- test/test_pack.py | 38 +++++++-- util.py | 69 ++++++++++++++++ 8 files changed, 297 insertions(+), 13 deletions(-) diff --git a/fun.py b/fun.py index c766f8e09..883062eba 100644 --- a/fun.py +++ b/fun.py @@ -113,3 +113,4 @@ def stream_copy(read, write, size, chunk_size): #} END routines + diff --git a/pack.py b/pack.py index 676fa26c5..a175f48dc 100644 --- a/pack.py +++ b/pack.py @@ -1 +1,191 @@ """Contains PackIndex and PackFile implementations""" +from util import ( + LockedFD, + LazyMixin, + file_contents_ro, + unpack_from + ) + +from struct import ( + pack, + ) + +__all__ = ('PackIndex', 'Pack') + + +class PackIndex(LazyMixin): + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + + # used in v2 indices + _sha_list_offset = 8 + 1024 + + def __init__(self, indexpath): + super(PackIndex, self).__init__() + self._indexpath = indexpath + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._data[-40:-20] + elif attr == "_packfile_checksum": + self._packfile_checksum = self._data[-20:] + elif attr == "_data": + lfd = LockedFD(self._indexpath) + fd = lfd.open() + self._data = file_contents_ro(fd) + lfd.rollback() + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + self._version = (self._data[:4] == '\377tOc' and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", self._data, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + + + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha)""" + return unpack_from(">L20s", self._data, 1024 + i*24)[0] + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._data, 1024 + i*24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + i*24 + return self._data[base:base+20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._data[base:base+20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size * 20 + self._pack_offset = self._crc_list_offset + self.size * 4 + self._pack_64_offset = self._pack_offset + self.size * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._data + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i*4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + @property + def version(self): + return self._version + + @property + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + @property + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._data[-40:-20] + + @property + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._data[-20:] + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = ord(sha[0]) + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(sha, self.sha(mid)) + if c < 0: + hi = mid + elif not c: + return mid + else: + lo = mid + # END handle midpoint + # END bisect + return None + + #} END properties + + +class Pack(LazyMixin): + """A pack is a file written according to the Version 2 for git packs""" + diff --git a/stream.py b/stream.py index 10bc8901a..44c7b945a 100644 --- a/stream.py +++ b/stream.py @@ -361,7 +361,9 @@ def read(self, size=-1): if win_size < 8: self._cwe = self._cws + 8 # END adjust winsize - indata = self._m[self._cws:self._cwe] # another copy ... :( + + # takes a slice, but doesn't copy the data, it says ... + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) diff --git a/test/db/lib.py b/test/db/lib.py index 738eeb851..35823059b 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -1,7 +1,7 @@ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, - with_packs, + with_packs_rw, ZippedStoreShaWriter, TestBase ) @@ -20,7 +20,7 @@ from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' ) class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 29b348876..6faff4695 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -4,7 +4,7 @@ class TestPackDB(TestDBBase): @with_rw_directory - @with_packs + @with_packs_rw def test_writing(self, path): ldb = PackedDB(path) # TODO diff --git a/test/lib.py b/test/lib.py index ef08c2032..6b25876d6 100644 --- a/test/lib.py +++ b/test/lib.py @@ -47,7 +47,7 @@ def wrapper(self): return wrapper -def with_packs(func): +def with_packs_rw(func): """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" def wrapper(self, path): diff --git a/test/test_pack.py b/test/test_pack.py index 0b9e448cb..a0c8464e0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,16 +1,38 @@ """Test everything about packs reading and writing""" - from lib import ( TestBase, with_rw_directory, - with_packs + with_packs_rw, + fixture_path ) - +from gitdb.pack import ( + PackIndex + ) +import os + class TestPack(TestBase): - @with_rw_directory - @with_packs - def test_reading(self, pack_dir): - # initialze a pack file for reading - pass + def test_pack_index(self): + # read v2 index information + index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') + index = PackIndex(index_file) + + assert index.packfile_checksum != index.indexfile_checksum + assert index.version == 2 + assert index.size == 30 + + # get all data of all objects + for oidx in xrange(index.size): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + # END for each object index in indexfile + + diff --git a/util.py b/util.py index 291855630..5c2bb540b 100644 --- a/util.py +++ b/util.py @@ -1,7 +1,9 @@ import binascii import os +import mmap import sys import errno +import cStringIO try: import async.mod.zlib as zlib @@ -16,6 +18,22 @@ except ImportError: import sha +try: + from struct import unpack_from +except ImportError: + from struct import unpack, calcsize + __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset : offset + size]) + # END own unpack_from implementation + + #{ Globals # A pool distributing tasks, initially with zero threads, hence everything @@ -76,6 +94,28 @@ def stream_copy(source, destination, chunk_size=512*1024): # END reading output stream return br +def file_contents_ro(fd, stream=False, allow_mmap=True): + """:return: read-only contents of the file represented by the file descriptor fd + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except OSError: + pass + # END exception handling + + # read manully + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return cStringIO.StringIO(contents) + return contents + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: @@ -93,6 +133,35 @@ def to_bin_sha(sha): #{ Utilities +class LazyMixin(object): + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is reqeusted, it is simply + returned from our dict/slots. + """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + class FDStreamWrapper(object): """A simple wrapper providing the most basic functions on a file descriptor From 0717775fa51460335976a046072cf5c492af2a91 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 16:18:04 +0200 Subject: [PATCH 0013/1790] index: added tests for reading version 1 index files --- pack.py | 6 ++--- ...438c19fb16422b6bbcce24387b3264416d485b.idx | Bin 0 -> 2672 bytes ...38c19fb16422b6bbcce24387b3264416d485b.pack | Bin 0 -> 49113 bytes test/test_pack.py | 23 ++++++++++++------ 4 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx create mode 100644 test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack diff --git a/pack.py b/pack.py index a175f48dc..2ffc64f52 100644 --- a/pack.py +++ b/pack.py @@ -66,8 +66,8 @@ def _set_cache_(self, attr): #{ Access V1 def _entry_v1(self, i): - """:return: tuple(offset, binsha)""" - return unpack_from(">L20s", self._data, 1024 + i*24)[0] + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._data, 1024 + i*24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" @@ -75,7 +75,7 @@ def _offset_v1(self, i): def _sha_v1(self, i): """see ``_sha_v2``""" - base = 1024 + i*24 + base = 1024 + (i*24)+4 return self._data[base:base+20] def _crc_v1(self, i): diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx new file mode 100644 index 0000000000000000000000000000000000000000..87c635f48cbdd3ef50ed7f14bf339382f9af6e5a GIT binary patch literal 2672 zcmbW(2{hE(9{}(xgBd(K*&dTMYnGWTWyziz99xMPjlC>~p})toMV67`B}1}TmM2~q zOyNC)B$V=>Y~wFNCXZJ=WU2ml&dE8R)9e5G&pDrSzu$ZB%$eWs-0!^?fWLg$V76@! zjQ4fx<(7XHr!VEZiu;rwkvFy2WR<}9M0 ziNW?u#9{x}NWk%Jk}&=fDcHZ8H0;~0S!1ubC&}!{~9egzD*m(EIK>r!k7i~*+JOuqz7{r{l8%V`*$;h zeTQ~A4D+ushU0(71dg$o?qUY>ZRRlkf+Roov-HpfiknPsQSgc&e(qq6WB`1JWKPan zB@S2bs@9j0Pu|~d3zptAo6i9t?h^kI1DSrnq$x0?CZnAHC2c>5vI8K!Ubt5DVi7iY z?A_8z*$9G-R#3e*0)XYZY(?$$*T}c}X^F@3#WAThDSD(2@KOCVt`3W24I`2RsnLBe zS3jw0nq2P#z07L^-wp$TW4myC&t@7Gr+ZVVr6g z$-e-=Eo<#hN2Q21t3Ra-jQb!Z=9wbXoA5o|tkXGKDP`nmyOsVhEBFSzNeHEyv<^U> z48}jq^E#hMi98}g93TD4@T8`>B>;(nSoROf%FTl_#jluKzhpcYt}lD|3V>@9rjgfirN;=VrfBvag&LO4B?q530k~0*+caMbZa7Y`tgx)%5tzpQd;u;Cr29|` zg4`9@hf42=_8lvAG#>GT`&3S$x6E;z*~pNXEd#Z#RO_xjl{W4P*G`Rh8+*|#F9jD) zzzDbvYWkWWLN)=2+6tU8v~Iol{rb;BG-u|EgvR$DD^I}Xt$J>yH>2(npZ;`;ap+T_ znF0E536B7_Gth=y2XN&VQmRuEmkqQ=~}@d@(%C z18d}t(ht1#T;EwyGmkzOP?_LDh^KKd&j66;5L;?~8AqfzuAk{?`@lec{4O#9*1v>p znw+nYlJ7^=)DV*V%wjyr>I#OB1M8oyQMA%DR=eLa4S{}gSy0&{e^`fw-dVq=QcpE!H4pW15dKd_d;q9mTz)fwH2@cAKACp2 z(fn9~Q7%3ZfN13_?p7)l=hMSHgo$4NJ~#DHajXZ@8?8!Uy7Gtdti%pQ4XVV&guHY^Hum*BmVAK=#l8yatThW*Cw-l zA0*q7;d9|z3{TR1LV}0_<6emXY1JSW9tQuv_t39)r0UeSghr*wy)K~ytmL%wom}|) zzBI=lB7L31T2D1nQDEV>+91(#F93cD>|VO{CQDY@>+jHY6ZJI9(SGOU(!DkXfXl`HYn@z}jw8u= z{7#8S{PZ4&STf<649oXra}3>5Q?omL(>2z%3 z72f@_1ptxy)RCe1UQ}tc#qSXAaBd2{;EHe_0IhK&fSj*j!TtDkOh!-c0d><`T|+AX zP;?oJ-3H6|xa-CHDhA4iboaYr7McJkX_{=;A~u!CHkB56{}I1>PLX%QJ_%k6a;NVw z@bp@dquPOdQ7(IQPVPbQK*O@|lhkvb#-aP;sLT0jd86!ebCf6}!;r2C_hk1|YUt&NPYZ&3 RQ*{1wQC`jG0&mm7{{mdj0G9v& literal 0 HcmV?d00001 diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack new file mode 100644 index 0000000000000000000000000000000000000000..a69b28ac68bd547d49ad7502b1b3fdd42d74ec71 GIT binary patch literal 49113 zcmV(^K-IrcK|@Ob00062002Xo4tSiM&CLx0ArL^}J5_KuF~jh;B*xf-HUPumk}Qis zJlMTkgV&dQd2=w!;;H1s42lU+BavHz@?cHyOVZwJGR7}TyyD<}N~cl=9hfapawdCp zS{ns65Fc}d^V_uBgH*<#(!)Z0}m^v=PtMR@`Q z5JzowPX5;mTI2rYEKA*3`~cm7Mv$8ec$}TgO$q`r3_#(1Pm#SK>9mtpkik3X1yUzE zf&M_P3vcg$H*ovld!vg$ijL>F5_uBh;4Q9ceng9i(Nl7k!W59v@Ox~n!&MT$hH{15 zrjp^DKD|{f?eZ+F`FL+-0XqdEJXuF@zzNapwdfe~uQytCM;U%2^99@ZIe)K_0eGBE z)q6ZtX&(S^ZNkVVx0dwEODOADY9`rg$Zd6T`5>(l&YU?jbC{Vk@0l~o@)njQtX;j? zP|0n3UDmSemT6_zL=tVvEBB8Gnc8f%Xi&Z9FlW~P-_P^>ZqM&=ba2>Br<;Qq#6zNC zd!E{{tjS~31uf zA1Mt4LOdA2F_?57#dHY^&~_Z)A3~Py{)~= zrH)kw6#|feEhgZJ?qv0xTSy1(TyqROP#D1M;}}s;ED-=4WI03Zj`vx={-uhr50-$o zs)u*aUS1oiWNa8hF%T!!){@Fg&$M3cNliGMYfMk6NOnqpt5SR%B#3M0Y~ge`tIxGS z!^TgvZNc3S^14$Qk z0}~E#oPHO2$_-SCtP=3@kV&Y@7u(30zD6;4-cOT`%%OZEfs3oIl z3}Nn+mVRt?*)qw+wm>Pk2o4Zc7lAk+!4W=zfAvi-Pk9B^T915m6onX@$AW&$rZ5JI z$l;XNjt@w4^mpv{RSRX_+^#4)<#A6%U=bl61PoifF|zKz&e`j;FFUz6?^}b5O2Grg z{ZSzCGc!qvX^%QK$hP{y+Zcoy{X9rb>Yb zaTpV!{FT<8ay`QxDm!yM0B_5;$C1Z9J?r->As-Rr0LqGlA)K5j-2={MooaH~HhJ_& zbkON7e)nj;l;i^f7~_)X1*W_bBy(V_eQRUWaBuwTQ~An1&m4+M#;=E`? z$c9f7R2J}s1P&549gwGqiQCU4)^^4h}KJ_)Syz==z6(xreS$?wZw>$K-ANtJb7l#}&il~vT zvu{}QH{$_hKM2-gcca{=vw2sDQJC9#$JgOUTjxh97KIZ^NY>-ZT;SYn)s%UP#^;Y{ zdxx8uoSRjY!cbz;#N?Hr*%2SNzV1zjhG`1x$`f{@se@rk^)Cs!JRq$vuD#3z7Me6o z#gqPV^p=LY@vBOL!NhU{S<8#WRbsrc-Y>oRvNlA=3TA8`FuT83F%H@KQy`zLTU??0 zH$HfMfP&T5^mbV5-G`ER;BQC@6ODbnK& znYB>x&K6_$5OwbTKDNq4BVwZRgmU~d!!`KldDGygSoaJ5gHm6K?lh$&Lh|hjKq0v< zMs7Obbp>1N3iH7~&(86!bcXyAl~4=|+0?f9=H(E>S?x<1JHvM(MwbEY&yGMS#seJz zK8i+3$kE+-u+(%v=g`u*JDGwjy>p2#8(M@r!sOG56}h36Jo{gW2A1zj~W2KQ6= zVN8^IbvQ^~^@UEg=7(O=jDwFh?ikWkH%?tAXQ=?WR_R<(*rg4I1(xb!5!1sz@yy$zl zZabS7BeuQ%X-UOkm>LH+J#DY>yo{{wU)cK>rpg*+_D&|N7$JtOTAgQZXjdC|lynt! z@7f+rpRQ91iE#Pdl8}t-!UWg)xZ+x#!*J-aBVt$=&9m{rj`n zw6^R|>|C$i>Yh~!G9H+$M4Dl~o|f;6TmGge>`q!tSfkznQ`VfGgOuuOK^I*^SpP;7<8l4T>*8AK@jT28Qowra0&H{xa2 zw#sqai1#PR7#v&^guG6*$cdC*Wtz3micVY#WIQwq&byfEb@VTK#c^RnttU9l7h)}^eh+r zmvU;+MslWA_=SAeRYPJT$ZTm4S%Vt%iAE1%1m8sJW6q7REX!bct~!c*myMBsdCoD2 z7LxesP`2?&@`P}7+~yoNLxhhTb@hgZ`(-}$G&R?`Cr?CrGyC2W`?_FIKoxv%>&~=- z2ALRCtt4!P{E2#J-w|jX>l}8<4J{0zw+HSj>Q(js9TkQ&o`Vn_@9X@4CItSN@>)Gp z$=xY;qBiL^5nY^gAG?C&bhh0NSAhD^Lbv^a(s6&N=)#~if zl~tXsi}TIr75@iasJt$)GkBbRQ^9V+FbqA@udu`cT2Ug*<=sFkj_%!DGVkN%oPadjLbWSAT0=J zwNwrbI1R3*jBdqShhUxQ3fCEgXc8O)*h!ah^^{u$5W&4XAT?yLRJ#}qQiY;E$k;QH z&R>w~T`e4!71|iItyZOFTIueB8!Ql=PUAu=>!7FwC1W7Hp_c%UtXzo(I_q`f2gh&$ zKefm*YK~9O(|<9QXsicu^QAV>c6lQU7FpC-yq=-QkZ#d6&NB7YtdmW;7#~6y^Jg~e zsR*yZ%lT^i2ioiD!PE72kzB&p>1q|89ND5OK@*n5@3q0cV!8=ueBehL#voM5r6`IT zMhDS!ID$Ryb6yH;8Vm2_8@Xh`|D?5HzaUi>o#&#hIXaIe2^>M8R zmqvTCY4AmEkmEhsm%vuI+HXY+w+SO&mmy6?R={t-KL^(L4JwolqPG)xoMn*94uU`o zMc4K#8rUcy8rSYzxiIk)2+VY(iXAc!<0JmNfEc5$+S}fqHci=xCWGtkZe463?Q#}+ zOGsH%6E$ICs{^SpuM+^e1agAPiVvvj(qUFLwkpE4r4@h=E~Qq@U*ssB_d!w%&wx#nlc#A3;LvH%T;y|Yy&z5y}+0f5Fw8qLF zJ>Tdb_D|_ODT!u-i){P~W(#-%_-SZ)xiWa1ZIZ!m+b|4<&-GIf$i)d%_W^e5+O)ZJ zhrEEH$doN4rT|J#nzx@QIdih@V$i>+kNhdFoTUye{M^6(dfTo) zci=*%l@l-%Gg3TVA%V;hJ7FZY0@5Sik{48#6@;icwht*1dn>UZ@Y{vl{Cq7AI{u9c zgP%@aVS!~oDsgSfebR?hvfJ` zvNlIBdJ6cD$>`yb+c2K2>nFxev=O<-oXa@+h!5r-N1_WqwJMl7Pui%RP35oarjkcH zd)Lh!w~8VLa=rYp;_oKU@Gcb$NgZvySgexd z;;MQ5`o*-Pep)S8o5kH?y;-h)V~lzJRnOntEf(wR+eHw3*)!JfK74NVeLwjA_9MT& zzWY5eewec7DJ!&L(&|z;$XO;V*cr=85Q>SyfnkjWmOk%Py!r5HHNU?55>))y>wf!g z`F_rWVbA-i=Uva|i+P0e^E5%LAB6$sEEA;xksnwsj79rG;r~2`{MVt)^_Sb*#pZ7L zmcL)zY=W9!rjvpD995u=p+CA{cCQ~*)MSUsN8(%Es9cx>@2q$|Z-kW@BdDel zPHN>9WLhDhY%dH;YlA`gil82}%<0-F;JI)DNsY2{r!bCEIR?U%2d*1eH!f=581Vqc zijozj*adJynIs2g;nmgUBuNslIi8i~6__2^{{AS+`*z@33k z%F3!PWpUtpjITC_NJf&5N2hDewaLM>1hL~GZmkkhctqNkdV55Id($Gk4hqMO-0fZG^P@onDC+B=3n3yn zqG2gw3fjPCdoWl8wanUJ7Sh;OhHF(~lw$VgvM8QR9P(NPOP1-ICiW1_Dd9$ERAC(1{0u-pS_wMU2fjKZVm|E= zk1%7Os>}y?#Ij8ogWVhpzG(E3a@7)iv5*=o|AMwyWJw!odx+DOLc=2R;h^0asJX6P zlKNBO#kwZ4b&2ygjv)M2)ccVzrXUjiPn5GYWD*CbA0)O*w!N1%7G$Gb0M07$wYzso zFE#EO;vmpdx>SMg?NE<~cA&*v3x^ZJ`n?$MdmAv&`WnC{Ir14-hV>Ub&(3pcu*c2S zt;d2wO(Q*)TBL+>`+GJSK3+IIJGg&$Bri@GtHtLexjLoKmp3=~mRW6*my-#4iY*{5 z0{Si^Kir`PdN28{c9Ht@4>uSu6nE2$<9(i9vU96@I-5MulSY=n&Z{~lqmx+DU{t^$ z6>4iS+Ec@LvX|_1Go$IK#@7e9rXHvvnlQU;ESVzdq4VV5Y+#7S2KHDQC3F_>b+Nk` z3gG=I#U$?6i9clm_stflTR@2lrhPn!jn2_d>J3q`oR3NmaFiPCC+H*kr06IKl`!}1 z{^V4($O22Xe{rNc{Pw1MasOaAD@z=f(%)(SR3zUy^%=btPiSdV$3}+|DvqI=jn@Z} zL&pq(&=d{Z<6`s}WGVjA%3|T)xtC&EpMEd^M1Kb#46z)gu|pD{u1AQD%kB)&|D~Na z0(ZK#a;n}7`OOzvR%HI6`)$&0z_2|pyF0aNf5QipHp2*?(bc^)1NlE(JgK3{Gs0d> z{sFYLl92F@0Wt85KvuH%8&3E!lmCbI{Fc5%m(57^ug>pea)Lz=E z<;JNuBoLJERDxsuK^bge;~qA47! zLAA%>>g0k52F^Ntt<6%iTNFJ`Or7F+L%y}02G(+M4FxmvpCG(DmfBBZc*rgAf+@@! zTS{Lu-KpcX!g*JoMF}T=!CC(4^6F!LT1o{cA%ML}16pRhDDGZnZ|?qX=TG8UYvZ|x z0d7aww(4deJCRsX%LSF1E8dJGi^Os*YKl5tC?F;U_bj+aVGZKibbf&QDa5iAuo|vr z;}zrH+X26ru!WM9#x~p&U+~h}F|MZ*VyzU{BsQUph7AH-Q+3!1(o=O2F>hh~`PrIx z+V`csJILOSVHekRdxvyqZSYz_GS->C=Dvyx-;!m2galdwwgQ^jT@ ziQPyQM9jP~YT$d3H#j3RAM+D#3NvC zxgF9h3-Whmp(P;9AqlPazNHbz=9U_j74v~T4osPf{EFugM}p(8%8^11EKUDz&nz^? z)245@RAJ)~(Ewemrp}7?*D0QJNA<3;=`3lLHBD#i`GhTq4j1P%%rVo=f_SEJv{7O@q+=Ib+VhI|6DBFmavbMQKTzeUw`&W0CmvYE{Y7GKe~p#l_g;=! za)k~MjA@nSP3q@PkHb4pvb3zj^5D9fdMT<{T8XaQ+kU;~hf1-6Pr z5z19~RgbX1vdsPg{U{?~vnP0*jgdW1!!Qtr=lEBgl%+%&fHfNei2<=OAr!gxxv>;3 z2PZ;uptn-vF%88kx}^xgaY?T1LJInj{*M2U9LWMQXv-S6aO zB0FL~pti^vcIs%&$oQ3f@Dangi+qs=GP#(!*)ubWW%ms7x^PIR@fl$)k*s&>2WHpWBDzI*oQ+dWZ`&{oJ;%R-XfBD7q8~f@pjt72X{EAKYVdv4XFPvS@=7Mqlnfss*<824q8Hz1CX}2%6 zzb9!}lvXjUl(mW$LO=B-tc@@C!wVTAm8wd918Ul!U zG#cHF2K+-X4ShKkXYI3h9p~NF5AY=jW$}IMhamLknJ>lD>mc3rqQ%lpce6B@@6DcS zl}Q>;vZ+}%bCVgCltrU=?L+5Xr+wZTmW4n6J`W~)vo{Dg72Lny><^st&PA_1?l`A| ztNwXQeETN;r`J7oe(enVSC`^vad3BdbZ}6R48P`XDo>*<^qbyJ(h{QCgr9$bU;O*0 zmU#0O?c1>PR{PfLzt_6R8#fO0 zLTKM;9O@;2qtQ6hO8`%!@w|pwog&4H{cVwjH(|62#Ug?UYX*yDlqA7qE(Hk0BnaI! zh(a-Px)p;*DZxFI%9V>e82RX~NC;sCMl!Du&G zN-vlO9%I`R-IQ68MA)og2sR?>CXs&+gt@UyO7bT_O)QsCCc~6u02>F)gsXn{lUPMD zjHHQ_0ZM2crZ63=z=yd^qiAohwPj1m+wQp61?g0Csn?BEU2}n}3VwA#}HpYKKS^&8{|&e<}8 zwE$CwO}z-D&P(^iNG?;km;i3r4_@R98c8W;X}Wy9zmJ>@+|Wf-`^+3-_CE4&H_WhY zVn`zFJ?*RB*a3NRJ`9Fq8d5G)r)~l($9Gd#z`mDxX%?3YveZwo2)&skwAQ{l>b~u_ z$5+D+>|w{p$A5isbo{&lYW6AUowKLdfblX0XC_@-F#p_YWTBc^OG;?~NYmK~*6{?t9xwlne z2KHHDMIt?Wlp&Rs!}I0seh^0cS>W$q!~Z1sT?)`h+CBgx>fJzdr6#p8(;jv00+G9@ zZj*vHu4tlqL^+5+2L!kXK_)@ITVf`&I7r}gp-K=d&;WZVtXbrjH*>>l>E|N*kw;B% z(?H@{0F0C1dKb51HxyCKniGgd>as*_C#-jUc?Zp4S0)wLzZmhN%k{3o%m@%BHxBa@ zrQsf#TIIBdKTqmv7OKYJI&j%8f&tq_xDVT9FfVq|b+rg*O)mom#<(vXemOe$s#kyx z2h|7`;WY?(gTO-Av%XyPI-i{OaM=F*^+dpuTSOoNz!nq~V~hn)jQ&08cy;53@>+gC zc@0aTw1z8)qlPh1T*DhEtzi$8+8k2FIVaX~u4eGtp}b2U&Amd!b)0_tvUAxUy$1{Y z;O<~LowoGa(YXDVtDzRnK%=&_5ux^#T}y38@46Rbad@2k@&8 z=iosoWQPYfwqfUT@M~UuRH?rDFdlGws6MV#x6jWz2BlD~*S~9z-Z4PlxrM%;L*%_5 z6n4v5w7h3S3ucHcy96vO`au?p%M&GdqNXv}2N7UR$$V4R@}4-&Qq-g&ns7?j5K^4W z9ijFsbki8|lP!yVz-{oqc&f?)ebb3=ZDCW;aX~o8R<{w@HXcx~tCnV+aQR8PBkC%W z*lTT7G(3&`7Xhty-{kTP_Mbr|NFi>c=Mm{vWFZX>Mhhu0`a*1NH5yoI6vE1mCD;xjxA-QSghu@Qv$@i6 z=uezwP;=!LqHc<7P32ZS`ee=!=F zUuht@fGcI$#JyUhH!my)CR^pY&G${MZFMW%i~`#tqp5MTX=ubB1(ueqq3DD;8#fU_>i&zI`*@#+s(8t`sLM z9CqHqa;O_B3ecv}@Gu-y$T-Pqt>*LOj;J3f=oxVIsSMgBsUz>Cm!6zTvW@?=7?6DW3Qv5(n|{sC4w)z!AiTE z`hJR1Qx*f24KyMsEJ4;$H$P{!-}cYwTu|lQ?x-{m%wm{Y+ffhEB-dbQ5eTpgS#VUq}%qdnbm<6H6IPt$*jixCpf1}E)3=H#P9R~tqduV{=jC6NJ6LhIVhQ#|`5Efb*MPkge*qDT zB@~h6wN3c+PH`v>+9}|21Ze=?_U};oS{#3=K1`32p`VNdqU zafibgLlAQ0H;D=-9KcqD1!To%7;yEYn`~JG-}+I?-o%O`u`3)Jy5&(G(4+7GP@RCy z6fBl=G{mFqde+(zlT7vU%;(TT-ZYkEEpi~6K29?4MyBNBdoBlZ6QKNtW2_vAEx?3a zEYo{Xu+d@>g{?g?ih%fM&>#bXCXxa#X;coMVghOI^jBJ+45tSQq#+5cow z;c*F_Q1l1R_#K{yoP(cuHn{xI9(PZ>z3%uk7`$i0!KhfH7$kx2*^6lw(zHhfEgLM1 zD=<)20VCnBLQ-)B*jRGaMyg^>Gg^|BHdv}DH`uLwhHtj`3jsHOEZVhdV99uBF5Psd zH{#N=<&I#G62mGEQbk@fl&h()w91Z9!M>IPv55dw1h3%&h3L6)Nqqi&J^fj@kTLM9 zXi^lcZ=>cBC2$!fP4-Ha$=p&JqNvsACCC>4LQrjveBbap(b9^RcrwwVI>yO$1_MCQ z;im%CXN4AtzLS7W2jsN!Dh(r6nG*zX6uws37ZUKRJiiO8axcKBl=gp9{;rbm(-iF zeeFpeHcuU34UFoVN#=3q@Fw2K52k_i>#^pQ--EN1aCwLOg(>7a0`_QB{f4wFNOOUb z#EU}Y;bY#kJhYZ!wh+d}^BSMAbmt6%v>4!Pu$L>&p#o4WL6DpS zM&tlbg#fA3>Hpdt4*Hj!{#ZO!g-;ErkAvZR>aPl0RFy6CQ>P04z3U*=Ku@a}f!59$ zwS-@T;pY;XH{?nKF23mYG#cIx+ZW?1C>NWcIUDpZx^Jt{Twa0e z4bIxVbD9P1qLDF=-4u39u;OH*^Y3C%0S>}x zv|k5ZK(f)}d;pp6eiBc=An@0xg;`CfBZ)I{W9h^ac8dUy4IOWKeX<#(i56VT1`1#u z_;@UkI(`saFcibz#=2D$-;kwJ*O9_ez}Q&Gl~%Z9y16f?SIe|G5?u=jdqMIDFl7qC z2DBFBI(DaN9SENjmNe^Qp?z!PC?CTSvep2YHppu_bKB4hS>Z_qz21QAN$qH*-5Ym? zeOgmB%fGC(6hXH$DuqtRPoqPsFdf7s6(%HrjmLn_A**A^mXBf(L#hejsm}R?v$Or^ zRvY?Y%4t-!Bf^eo>wVn(D|9%excB~9l*Fi883)t*&A9Y3+VpEtU?k^z(P+vHx0h4r zzpuKTu{gNHWDip(g~;oPDT?5JVj>aFffc=EV@>OT{4Ma$ns*c$zH9yE@MUZJ`TkLH zyfu{Us-Yh}$)=bVLYWT^<<^Y@3P?P=dzL%z=&PfHKa1v^z=d3Qfx>5D{495J zPWXU}QabRUOce_+i&gd!^$a5+9N=(2qk?BU%4^-GfaHcH6sudE~kouNLBT-K+qNgG??$dP}vEWQKZqVwQwlFJE*yf zdFpUl6>Y!(46jAg<~9_z6!ahX^AE9_p(f^oP0=kPP;?rXC47p;K>*xN;{~S@mJuWr zFwJ$YiyoOO-gcSmT5MXWh(q_2jp(t3h4M_ubvBxt-txmFTdwl@9?fP-7gRN!F`l}x zPSx4i6qU1ei5hp|!l1@bzKCf~m=63VMsDl^uokQ1{M_LbssY%}13ZEkhYj>Z2An7A z`^I9jz=s{MNY6UrsB8g7)!g+Y<<0Dusav&3M(?{HobwC#c=lc#mpGD}U@6Woa1_&d zfXv1er=%Xq->*inRe$)<>7R>NM0$Ivl1JzFpuJ!R-Sb|*CNZY%v}LUZJC0G$(RJre zls?B{j*TK>cBE{jFw@64 zD@QeT-M9V05S+N-a4;M-j_vsQJN4w^0kV8Etti{u+dJvxmWEV~Q)j zbdKE28`{RSly&{m;dU++Bt_<^@cbK<0vRuLRJrxyJ8AJ*18jrDH@b`v2MO#K@>Yg; zSXSf@VXAJNNh_b_vOPX~m(L{uHX302FiuR!<=jM04#v#kG{Ja2DhbRc1ND$%nvsZI zFPPu&dSHHpg@sYq)v$f`+3AkZ5+8qV92^~16%*$ZmfAl zA>dJMJav80uPC>Vso$&>JrZ!=A`e4$<^Iji=T0?vgg+iK0l4>NpvRZ$`ZsK3!fO*y zix>CoM=q%cM`*bSpfKh%scK=PUA0zqTgY`Qe%QGfc19q>9k5A1!$KiP0XHn5I*O&k zCu^G6=Pj~L6e0MJ}d*+i$t^vWt``2f~SZlmHF4)Yy3iSvdi|T zuXNI@4GY~Xr|820quj}slqgi~f6W%qdI%xiFY*5@LOcQm54C7kgU3=$1qsr!%8jl! zb+n;jO-RR`(}kVfiEu!Gmsx69=~hqkD3K41Nbe<%=JSaQwU2AVx!H4>Vi<915VB#1 zR~v38yba=ru1s)tKINODwHsV#F?(zai3g@Zval9OUHXtg)o)pps*PisfeC4fj*;%9 zt5H@#=yaGCQ<7w0T4Z67N}*L^a74yALW(&W_4DW&&x)}^iM(0Aa^ui(7ZVtDlqJ>n zbU(`1; z*d{0HBr9>JlHgJIjoKF-=WN)!a4y=tQGR}6FRPlxO(i%qU4R9-x3||4j#HdnIZm_5Ocu>$40ABuft4hV zFql_8K>APP#6hrWpoO#RccW@M)JuQ5T^XTzRBL_6&=hu=;I~XV?qm|n+w~YM!=er@ z%+O{pF5^`5S%btWU^dXI9eh8*D-MfglPM?~!wej($(hA0Fl_Ol!(XjB(t9>K#>zE) zk{v^xF7IocA%`w!tn1OqZ;7az97O|F_h6s%*zOaHbCjUxn)rPOG;WS*i|?eR&f+G) z{1jP7pqHhL_G2UIEwMB}%#k9^a}f$NWr0FbVK#Ct{1Gn*g1B)WI8Y(LL^T%uqFldQHg_m9=8SD{L=c9^{o;lcMsN_Q!mV*}A0+XhYZCy0JOM#Do?%bbZXe zbd()fX{(|PWGb@NLFw#s>H471i_=Y*r{?!IaHyZU-uR2e7X)ZeHKdlqbuiDUpK7&^ zJ4vE>rY!U-er?&~_b&%u@h1MW{b#sfbZCk+K7j0`+5-IdSi%az%*#hNTHgW)FlTSV zrxv{bx;`IEg>+%v<=vG$DPTq<_?qDk$6nMoI&{zgP3MPp8fDs27^YU6X;FeXULv+# zy=^|zDz?c@3c3>OQb8%J2h>(A+tZp=y5R07+0;f*IqkN3soOrwuvOX3zu5sxo>T1y z)DfsMztWS{O_g7jE=;_j+`K&O2A((g?3*S-CF#?}U^`YyHZGA@m~WdWR+7^ zZ*+=+dl4QaX!ENYES6ZdV;MMXZ`y$rbjWl2>0wJj1-aN;uL#^@M7pMbGTk0*Ro6&A zU)|i7ie76FGkFK9p07!KIePK3Vy2mIED*G1VR*W_H1K&}$-vmhi@zcGMEF$8K1Epo zY@gJN`Ou0U%e?INyO&p&PW$w9*!fkIo=l!js49LkIEkr0nl-7okj9T##d%W)vm5wE z(PNNkh1=>%RUOhh zNDlMP;3zsK)#B|K1McfMI&~1)(v1_~D*}G`x)yRO@rTwh}{ z*)^Kt0j=JuT$p3RV696zN7bZVM_rdrLp^{?Vqu7?rb;W19g>jIy;ay<@Vd6s9}F+s zJ>@}B$*|LFkGsEi^6#T}gW)*;l1CBgOLsJQ_444bXn1jU^y=t{LWhQ$m_;jxPK8W9 zp{>Is!cV_Pp;VNYR4Uz1b{Z{VzJ;zK_c`x~Ep)VbXn|x?Wwz4PQ+OV)J~r3FnUuN9 zWgOiFmCjIDO#(}*^3bE9v8c#Ms66lV#}sUK27O>YJvSh<91{kh$O6Cz1%c-c z<+2{vsWkLW-7K9&F?$p>p`{X#&#%z)+YCEhfPdnHZBZkfdWl@P!Q3n&F6Dd1dhC@% z)iubqqfauC@#GgTAX+XPy1we%Gym%FqY%ZMld1li7CWCKtIuHs)X0_kW~4d2J~VE< z7Ir!i@jwQYne31gzMUt~v6&rJvkbKJt-`A(r?uFC>k@fQB~zX?=Gpcx7?pURv^+9i z4`aOXLp|hQuzLN2KX}tPbs~SFPM3A942CLdc}GE5$=b!8U{Ht!T}jm~DKxnz6J*8n zDU~)ly)++>rK9PP#YFF>9Q^asVpnK)w6~WIjK_SY)26tG2irb6k62LLh>_+^WaAEMVfup`95OVzr%lf_>msCV;L)=_nQQ*WS}Jo#S>lD zv_AC4l|CcO6aDEgkU4GU@LRn+VLCzGt79C@mn*$e?U!U^+R~&cDawrDk6{!}LH?j} z<;Fgj(cwMJ8tSM9?N|8X241Ga_k^xS`mH{_c<|mT7CFvI7Lw0F6%5??4D<#mT<-MIa>sKWaUrUD>h>1+sH zkIB7}<4BdLEjM4!I2-i)$o)K3mH(BLTIrm#0Avv?8&4)lzUzk}th**>1${aCYWA1K z))o$puh*ua0!abxj|yjfuUKmZ;Os;|JL*g?V6dx4VAWs%2AJ|JyGq;-J$5VZr*m1| zDIyqhipwPI<+}2 zW$dpdp1LiT0!EL33xHzGdeNY2>GW^YlDYng1OiyG0B!WDYTt}BwR<3f8% zn?N(aD~~5_K7-y4VaX~S%1Wg8U@dcUBcnvgdoT$>xj8%F|E)Xpc$g#Gd=g{Y-X>3e|^d%unr_mp5%wa?NSQcO1Pe*AYWOP_*gjwgcr8cDXse|9Ojv6%lnClNU!P2?HJn7z+@V;>CJJh!?G~0l(>orm zTXa`?ejFnCI0jamISUZd;MR>d4!^>TwJ6E%#`Wc)H}WW=?5LDXJzVHYY^z)Pn*6`q zZ+%2MMlSr`q&f~6-qy|MrZFe?nD0|VsRnaBk0$QCZZupk;~2ycJiAKYMoMG*6F&Yy z(ZS{+JqWCyap`)8!ew8y2HFvb4NI<3EhwqX1T`gw{~UR@oCjV2dmqfuErvxlhZcdG zUW8GrwUdoG)tvpV!O~seftbnpQsVP5p~q=V*28fN_kMz}j%DD00Q3AnHtFS!jp9#l z4^MqI_|bPDE7889%TRX6$+vwoN+ipVH8cZE#a+~0V@g~vd=>4eK7S-P%5>?_u?bt8S$XBG4Y$Q}Jr zBo2r2fG~9V>III64(_Cf56r%L(Oe_4n2Vqe%-|DWpC%5_^3z(<4Afe<7+h3mDfcOz z!fvT6Tway2{NkiD0sZITpEjs-F%=9wI`aUG(kl;Pa<3l(QHJh1C=9<$WPJceBY2wD$w{`8dvBEkEH&7ul9w&n^Is2FghN&ZoIK;cIIQ3%XCW8JF zjcEm|h8SYE-9Zk)RFs_dtoW8#`>(hzh=1)%37>VJg73b=*UC1SQ-m+Y;6X>_xtoVc zt05qlPQLliNwzA*<^b80WF7{oc7SQS_#FTP%AlcF^eJg?3OlLv^S=v)m%eeE8JU;9 ziPh@QCXu(p;h?R2Vdi(Wlk(*}Gsw2#K^-&tM)#3}>C})yTKiDw&@CF{>LIrv*J>Ae zns7r_d$VAg>YT2);cJR31mBGM#EXGE(|fA@+0?n{}k^l^mu zV7fgF2ai6q&pHk5-Ud`snRD8?K<1Czrw#4q22|stWq91$9lygUHbWfi5NAWDJ$5>S zo+i2a9gUCpDjlar4X?nSXjUltRzpT%iWRIWxj^|P#(P@pfY#grEWp{>`p!mdF$1^3 zDx?cqq1i;+6O5#uRYK@_#j|MzD@(cyYZc0rLCoVhEH;W(dcqEul;dCnSb0Pm1(x}M zJKvpy=2>&Cb5f3=blh+W%#&SOf%%fmmsd?7dRxD9%TWo-oG4ShD9(EmEO{>t!u&k| zzO+c=GM9Sd)MB)jyLwrQPp$Gz8vk<7+`Z?mG74)H32&m6({sj#jNtF?Ja7|--!(Qf zv9N-C4tg~b4@uM?dqlSVA7&k5Lc1w=oSo89Yr-%P2k`fPibG#qk?O32L7&EOY~Uu0 z4TK4yH5U!kM3SrMx8J3$8y!B#p4z+S|GSi=ua%Lg;Bhfuyo=|k&xJPlQ%9p1daH4o ze%)s2eRh{kj>l};zU)@x9%v7f6p9m}hj{b{czsyTXKyf>1}K6UKDNC6kUO0hC4wjG zI7VCKURfbn1#4ScmsC0m8XgIq0c{92nndMM)ZYb9$*YpG=nQ{)WQTEWJYg;a6(a3I zjv1vlpcR~K1qM-G9-Gv2&jUNdT4TTuPq5zR4r2b5A#@uI4&h+6gm_)t@D6{m5O2)F z+VUzWggr@HkpN6>! z5Tp(oAZ`jbK{o`2Kuaf?i$rQ9mDC&d-*-n!vLwG`8|EO0b>uy~d+vp&9EnsWkg|)L z`TXiO`ZzkmCm~@Y7##^2=T!pirCR<5u{~bB?8vMJ=u3>%%PQ^55{Q-N*6keOhAOr! zRz)FflnZ0opQ9H&DGP*!Vmgrey@7=9b~^fNIzZ}xcth(yi1MgJ!h*_*93LuaL?$6& zLR$PyAVU|r@?6P`C1~xQ^GmKlTC-Ut4p?A4OuXWn1;{`O+FOa{f|l+$!yh54!I~}g za;zkRd+mCMU6+VBItkdr%juq(!y#7H5X&=d>aAJ zf?4smlj%XlNCR>RGKT*FBFZqV`FcN6IC?=J<~|h8-zxzY@kjK4X<-Jz9q&l$koj!% zJ3`h^J0t#pU4XlgCPP;1{OmqOLK@IJ)kC&RIdEF#khBKc!$r$=LC4BP#cfw`d3Sw1e4Z`3lABMn>#I+*u4pztzv^k-eVHvTZx-`& z#O{heefd1QxVpTWeF%pgMk*O`6ALlMr%eHKI-o}<%kEf-KFb}i~UN5>&E8bxB8QLP6P8e8} z<6to~ge8qRgU#5HRU%~8G06&>LqsS$VziAA6ef$xdf}hg*V4ST(&uu&vGvj+UNsb) zrVhQ&hkY*W@t~PvT(wjC3)lh*^@vq$SHM!COBjOymZhHzMn&JIa_mbUbp9JLgXVBC;kOu0fugjdrz^p1&t}+qx-0?=_coM6! z%m8{og}((PL=S0GI3BqcT)I6POqE;A3%^^KpZzBMa9rdV)wN z@4oO!14)4qG+%}6X$2C|u&17$vNvHf*x|TGp~Bcfj`Jwef*xq_0ClJlhcbe~ZRh~d zWcEqtf)z6AxEi@VyP0!!Z9+Mn@lH&q+N!nqDbDP@=D87XjaM~Ne=_w)9vW;1qkJz$bWInaGl-TP@AP3U z;3e}m9A364eCLUvVv|3UP}lD6E;|`v@#Az=xp!NVygsWh6ZYyA`(Yq^eOABLk48rz z6Kn%C$k+dbpo;z*9Ce_YxLJ6dZB$K9+%OQm@2?o`5d~;3RVCB|f1?8LId?4PEMECW9^Nz zF`dK?j!ri*@t_k(Fes1r9lCz=v;q0)PBG-*?fVaRXNORR+{9r>1i4j)@M*&NEK1eQOPthil?xb;&NWc+{Wf@U|Yo5qv zOWhx+iOoFmTI)cwWEk5t8UM4RaYfgdM#KOUU99E1k}mNdpDLc`?cjieQRqnB1fnc#+HMsPMrE8C;CN@!iFM{dhPzLE`)M#{XN7{voSbkuwC zA#}n^Y>VD4<|)_wT~7W^JDeKIPM49giz|7qi!8QM5cgije^|2n!MS4OEb*eEfzlL= zHg#`^IuSXN4|;aqfi0Fj{pw8?j?q(j$byvEt(_TTByGzbFz<26XRK8$Vq6iAip_CP zoaR!qWfsE?rG}BGeC4RT+yKfPh$7Brl>_L3Iji{P^ZDhHAA8icvBHoqi*;M)xgMfydfoU=v86$}-uFU9&FhhK@gc78|H{n&&*-GugE1>`<=U9< z3famI**1Gwj^O4(5N#V|$fEr9&2?sy8}cve3giO1EO?xSQNc>XKoC7=zha=58Y(5} zMaijzN`|xBKYXri^+0sex6UL??%pqIOqe}u&3X}yXT}ue3ztg_TqYnCFfi?)LNLbJq zuS-M2En=1O5(yA;b-#!Wxets@y?n@^_i+JPON&6Ri(b zvzTP4oW+O69cSGrCEe>)bREAX?0a`myRfLOm!Rsr^zA9VG}6^Wwg{p@MA^%I8pMpr zElOu#czoEsit#Ri2@Ds%Nd94RtekDiYSZ9ZkqMkgX|LSU{*D_T`Anc6&LKe5W1UZ@ z@ubd&)x~*w*X{m;uVtytvBG|~|9@PX;^Z_NQAaEG1J#f0k-isroSl!)3c@f9#?R|h z1a{HVH|R-p;7t(kqiZu4|D@7(B8up3Q{jR_#xoZr_i zN7^$wAWjhJjtV&FiPtcPY)ou=54O!Lq67<}^n*k1l& zcO;P`JF3MC#$kkax!D1DoV8X~&IK#OgFv^{M0(u+Y$ zl*K|KRg!9)zJB}8kd!RSO6o1Jxj1~xH?Qv->O_>X;3d3%b9?t{{Wd+pg^+w$r_VD` zE&JSYP@7LYw@@k#1+OgwftEYSo~Op@4o}?f+U;JFVG0@KnlsDSqoWJRRnvBs2m68{ zRH;=1Tk(ZU*mPydHQJK-T_p@?ZaXavFld#KmTRyI7l&TKR?LE_R96?U;m|T;c!9SG z%5)pUF%M~@pcYHWSzVhID2H2W9)-JDJ_C3K@Bg{I?cEdF5EkMq2_+MCr#W7|t9X^m8F0PyugJ9RwEEE2x`Nzq)04R4fwz-qHC>Y^O$;eSaY zF8pC+$K*41dl6QsA|Lulf8Lr@ZYNY8YqIQQZUr*GVGop<=P4#yThZ3obkT4F_rg|C z2r?$!c^KnaN>Jp%u!cv0hoT|pZ%TyUg+wW9QiN3UkFQ+g1ds&*5RT@C9Aj-lCk#m$ z!>XCy(!8jcoI;_g^s*jlnW_#SP$FHPbAv;HIe%mI6Eejeb{Na7$Whrf zJ?()c`wm2d-spTi)Bmu&=fK#;>g}9O*xuXxE^E?nY4Rw6;d*YO4rz=6NA@M6goPiB zfS`$w-*+a)@wZkmt>5*Gsv0`IFryQ+kq4A=6CX~;5)x83V${}N3uTGhHSFnj2dzV^Q@iYY$Mn_|2- zd_tkcIZXuoFcIPZLb<7c7CqeA8Hv?h|9%Y%Iyr@(=ew8ZOSrlML8k)FuHjD-F@E3) zY;w3px8U^D_v8vF1-#^=M11ry%@81d%yj#xxtbn!lm!Cl?^4Ha$0Sajt^jTv_(e zm^ij^TdQ>*JDtv%b{wC_6JsJV=UU8YIy^uww2<+1I5h0967pUPxiKc)YXY2g^fIn3 zd$OPCTo3B??LFi|59v;a(}6MR|6&sAeWzZ3+hmg!k2!-Kjv@h}uCGeij^eP?hNzVdoXF?zP$df&>Izbfhk0cMTfEEKHM4nkau@pIC>@(u8d}yAQ%G zOIAQ^$86!dOST30Qatbgvx*ydh>BNX>O-)F2@Nrr-c9|4J-B|#SrQiIzo}O))pD%q zr;B=`nm_o2Jtr71$K~ zOas5{kHEuN;iur|I_c)!gfT7r2MzpcIv9X5J@f7eexZSn`vY?{aV*q+Xvq~T_&c3+ zknD7=o4z@At_{0q+I#qqI=D3$ShpY=oorlC<|hrjYYi-$BKb)7_fNu|Tg&c^bMU`u z;Mc|&yx16h&NSJ>&vd)`&uQN*8(zV6y9&Nnvicsb+f{=M16ym_BV$m2>vq+=>rb59 z{^VLkY7Va3RoncP-h^eFfL;DrC+EoC|O$V7@pd;Y2uX|BRHF{pA3D{hB};zc!zf^50J{5Yrvl&nfeX z{qIPQu7F|xsn-gJtMC&ZeJB#c$m0==@DhC^=J)PnnHq~==|hDO`hK`Ajk(Syg2g$NqHb(%>Zl(e@xWdEaL7xDy`l!IC|CzFLEt3CFj> zOEQ}ffmWfY^%|U<$x6fw{(xd^wj)QuWiznRaj}8t9m!gRY2Z1syrmhhAY{pvvgObH zFb0nxf}{kIgEd@ST1PV@B8FaIxdb{C&rCi8u~d(;!#fO?YeA--=TMu>-1$8~WPMkn zJ6P|1sQ&?)@*rJCeJB> zQytGzFIR}A-xEq?XVpAh&oF%x1~&_XuyPFa*q<1XIAq?4WFpC*GdPa@XP!n#T;aF# z4tQb3B#_k;R6z>HESh3^(u%H@{RfkhzFiH>JIAnXnGK6qDT;V>p;BYdT&NrX#}_I`s>#Fl3B3yQpOr3nCXno3Lo7AK_M``$vPY9=Cv~&WD1Q>*MOd;Ry(g6ANT*2fiCL#JiHW% zj5A29Vh%nSEX1;K3O=||F#*#pMRtWXpB3}^US2Bxys%0e47{h3d zt9G=YV3Wd4BSlsXUKmt}Ioi;kIWfBvbvaPpN;TNEZoa}U7RPS--7m(Si9K|gEgQ1U z&*mL6mQ902HgqoD^EzT7XpNk#ZXnyg`W`@XT89C#GsXV*@4+h^WUT#r z{_K_A4?)|JyJcGe85}#Mm}RXJcs@}_)4`jAl#NEp2D>d)VCSG|;jrHYVX#|MFVWuQ zP;KIF<5Z~0exUeMGtEL!$W;^KMP0JuheaG|(8d#buK}E8ge9T??}3_=g$pSy&jlyN zo{_kKz#}n#8Fy|yh<$oN`L#fYx8d-wNaFoW1+}1GYlnfq%haN0q{gU-Ox4guL!T8N z@yS|6Er(ceN=1E&8?$a(1Rj4>vm$|ovwuSaPLem=fIbFu)Ht8qGjX5BOJ)vNF)Vi zhZ?+W;`6AkR_aESYzFpdOL7|$6fCL23zBFD=S;!&cy&Do0|*aIh2a8%c4{sN>D)X% z8CL!1Iznm8+$7=a4fs*I!K-S-QLB?bTR$u~nrz?rv3@*-s*Gbam93uE2%*B-VNRd`QIzZ${BN01j&Rqjj zwyvpM_-VY-rEkjKZ<~i4r6LZz5JbdB9}VP9s$=weX0KK|F;29@blHvLUHy2^9Kcb9 zfSw*OwBSvO80r!71Vn(X2TZ=+nCj}fS38;LLDSKfk##!)DR)(Xp=p1nd8Q|1sf0cy zL7h&YUaIW2iNXzsG6g^B(C*K6`0XY;P22J!6Z3R{P8@IeTr752J`3Cz*9VtJN2HL7 zHD7O(epmg$#Izm5!ZkGYYfzX=+xlXTYA2T`y6cXns!|1blp}MYzI(_@8S_4U_~30;qq*LV(P&G~4KY zb@~UVVV(1QoJ46Lt4Ru{hOVEAbA18by;5)#>$SZYI!mg=&~>6aUvV$j1yHnxlA8vr zEc?>plfv1dli3j!Wm&5%zN`>kp70ooVJMBg$~so<_EAa+!YWIs-txErF4CxOgnB|O z7;L@#?+&=P{|9}nf1tSk0eGB^S8Z?GHW2=7|B7pib+X|&PP^s9j16!Sr}bL9g`4bs z5NL_EiAbbMQcbd=|9y9)EK81*R-Yu^bI-l-?noW?yU>SomECKx*;)wBd+=g3`VoH3 zrBHAVuUX2ZLeO#dnOMu1ui;{Ld3pYBISpTTpW(NVyt>pq?!(nq7}# zA{7bK!rntFq)77=f*(iU^{RGuY-h{q5hus z)v1nrO7?`O1>Zggv&C$FJ>U5@`wRK-`N0#$3Z$}yTg(@>NwIFQ?(CNKmx_+~jMaH$ zU73aiGZ~NIbGK8LRx-TI{bKa*xZ82x{fyZyetc3QhJMT}D?cJ>!(t&)(pXQ> zU*|HS?1!jHH5`hVOFOO@N)%SS5&v;qbTx`~W5-Z`Ft#OMbKJ>@djv({@ONvShUvr$~aL6^o=J!zY-g)CYnZ>ZyDlNezc3XFEEKi5K z_2b&e3omDHr{Qmtx3i0Y_P7T@;9rh@A=^E8@-vKj^*Hk+4%eFVG6LGzLV&{&NEYH9WTDmknyjo7Ci`IOkdjJsWa zM|B5{1>eWkP*M3ER4cbP-l@6*r>o&7qe7$`%RLhPj zQ(2xc`&HKlq;lY&hg9Qo9sF&3OMEZf?8ogK?EHp=n$-Yv749Zj<4G zHGa_5~zZH`xrB`n|qt|2hk6c;MWQygN<3MG$;{>UGSLwjd?^`c&B zb;D)o+&!piKWXAd8Kwl=gw6s#MW~Q#UHyJCTZY#QcdT3l8Fh7r9bM7ubsIGN`MAed zeP~oTTfTcUU68^*-A(|(lf#+fXfWSyjx*>~eku&tYe~GRY2vXu1$K($C*r)aZjcqa&=-bRX#f3g$rxy9oPDor@c# z@I@5qe?+q?`Iz$@uVsFvp#i*B-@%Fl)y=r#UFvb$P^>ZUSK^S~$8BTO+uOr~R!Tq9 zIxTJasIY1De02k7tZ~*jos}1Izmb+#J&2!ZOe0$X#RT)hU(-@P} z@%L`u9|4KlfW?nR&1K1XoNzbm0WFy}U8V7?tZBLx$`s|O)+ra$v+GwdqJOK(LU+Ft zF<<4IZ#sW=hh2HVHj%hy@VMq(+#O|y%D(=p76$DFPS&5%e^wc6Q!H$-8sL=)uhpNw z!0i^71H3D>XgJ4F<6H3F*G4dxhc(JunCMg2hEb`osgwai$O_4&u}JL7c7blnwS<*sp_5Y z|K0Dddm@u5WNbW{@v~qLNg=^;fw6j(Se6RSL0Y|L|AO918nKX&G9fK1Kn{XllE<3K zxP-gYX>`Mfv)SMg=9Pt^6Cq2zdXu`baq>t>?R*cH z8lIXlgqbMdVU@Z1+8l^$?8J)~hUV+YO@I2E))0>q$4RD~-W$4+dw?*xdge~!dXa)D_>p?wG1~{-x ztCC?YpPH7h-EbL!$h2NgC)4Pym&!Op5IhhDx9kqR<*W+%;yqQFy;*(=a~!+}bB}HC zP5?)^-~ArW&-nT9=DtQYyj1-pNK#{z1Ku2``a5H`f(__|3V$DgSq43i@m0G0=wy?ABU`+hiLE2pOi>hoTklv zG@QW22}~b*%cwu^`|^5?9)w)%C>QTZ&#QRXUWd=QNb)vRb;2=63`MKqMQXLSFAkBE zAtk=6KBTv2)Z(G=W?pM7z;X>mZ1E!S4UwXL!YbZutDU+~>x9KRT0xWHqM^`dBh zTh;5a0lQk#F3L34?daYq$Pc8UR8LhD%sN?VgJ~BVu;LFU-awP|l`PivzeBaDAXAYA z{{d7T@(#Hac${s}(F%ev6b9h?J;ecUa)dsBmnIchR|VZ2#&*;}PSGVkew$E3>vI3j z|DCZ&S4L3<$MX$~N0DQGIpB#!F4Km#F{9biU|{OBf?8V~$)XtQ327#Yy|B$rc4o97 z8!-U?rg8SEhHJhEq3F>tx2$mdtN&%glr|Q;Frl|(t5$Q|KP}E6<#4^1CAdnfjj*JE zr(5FrmvHAjBuU~{G8U0hy7JumU%d7POSNv4yc~F(ZBV;T12GV+&tGBb8aSm%o2yU+ zibO+^E<%>E_s*-Bwef=n@$cB@zy(CJ^2}H}JJaJt8Ycd5f7~C$)7}1299Ul%B=}9T zm_k;GPmq)mUaGT@VUFyBb;?mpTp1bokiB(MS9xR|QsS8^u=tAU!f5cxly|0xGlAxB z86oYq8`gDa0KY76(7Ix?!W=;)gLFl;bELHmzC(C%5gH&Td~kDs7#o44Y}RFmoOj!i zeU3k=Lg`>wquu-;mJfBdO-0nF=MBI7R0gfCcB;7Xp;SWkU#sTYq>NQ)ooR^adnpBB zKa={TLfGB zDJa-N>C+K&iI;j!B#GMo`|jSSwic-%dgJW4J2P!1bs{9(Kfc6w&o`6!A!y^X(qbQ~ zvHbaVKA+8DcnH3Z=TkTjf_@l42%Kfvg7Fo|jpkk%4boVUj<4b!%T`43Uuyx%flmd9 z&$ZxQBn4<>4w}U~f`D_g>!)};eaolt2P^!>YS^-DLkbs!(`(QYTVcO^s`L_OlM22< z+**`zYN=U<|H3rB7smREGCE303~1wPW#f^(~Odb@g69Y z?WydIV3#VMLM9m2%V4kv(T|Q#OJ(sgY(rj#Vvo@PI=D+A?sBR&VDQ-%4A(712d!UC z>^BxuWsVwqSSX-xKYCMZ6mx&ZgSet5HIYISEG5*&Y?KVX0rZzm2er5bc$~c&Yj4{| z^0W0THZCqDT}77MIB4p)g(A_mP+3w%DRr(52%213gej8c!5crm?Wz zj}|6*^BQIa$+NL>W%cLt{>RqgL5my^lTzZXNSMXM2{)1F)0lV(xedbmfTSTwvM37U zl&pNZ^{#zN>^Qa`$cApNX`FO`#32$VoV%-8mbn-XtPjTA98D*drIVL0NexPVI3>Ne zZ{K!%W4-m2SgZ&~4i5<3MCk(~PyG^u^T~1mgkc?k^D&bA7J*MuBh*i5GhP=lL*E4* z08v7>DGd@Y48*#?^;RGadkFPS(>0Cny@a-roSpe8Q<}X364zNuY;x_TaS{?CY|&oZz+N_Ih(f3(Z3CCs}n8p{-EIsUJvEluia0&&YS?* zNR8#|7awQF^o*^wtn?v*HI@ZK9aWFIs^ZloxHR8u+cxes0U}80Wr3~z-Cd5yBc@)Me(lEGl4A4PGv^)@z|fP^bRqr|7dO}dsNmpU--lbA*^O=yrR zxDXepoer~^kXPVaoC@r059AC|y|mS%&hZ=a4e54{b$Il+Daeb$rTeoP1T7C11$-7$ z`raJkL}g}ajt4dTOKUk9T@|{tuBJ0%VwuzBd|;3_UFP05zIW{$fzn*kA6=2-&bMO{ zQ0kK7ZnsUQ5e7qV z=#R%zx?TYV?`(jg9&#w?FL3(ngB8%^B!0qeuMRsg&>R|M_3byOT;F=An-iDT;bx7jfK>De# zy|=xpb^?eLXWbxZ=r<5ZLg$t?2<3%NRmoW)7EgVXsbzjNEz=kqg9R44GG*#hhf@|F zluz49LgNb47;?e33<_upUX&hsK{GscqVj|tNA@93)$rYlBSiNV?}PAHOr}<@!N>Do zMiW-MGPdnf3@IdTVlK|EK$eMFP*LJY4L-g)fg%KsnK>!(Aymk{2iZN&W*)#K7koKh3E&NDb!A|bpT~3&Ef#8U9F>XQ7%JiaiRJW$)f5iian|Wx4+=ZSCO0{pO29^OMIBPp52auNZT#%Czvg_4Q+oaR! z=sR}{H3BjP(IQBH>8ee7ZE^%fJtqrL0#w3y2i?yk$E3Ws;{cNFKcrY937vw)G-btM zhIv6_VC_H*_1zk*2#1tFjnAdiCU>^Qf|HeiMV zRqhy0m<4Igu_^@(`z`}I_YXN|K%>DOa^PLBd-h;Li~TVp<-7Fu2dEmrD~Za&C$tdLjdSv;)SYqJj; zm+-%ELArte%mdk5D^KTqEVjX+h24ajB>+-zyG1Jh)Ro+}`8qy?Rdo^1Fbl9EXIjJ2 z91cU%`N%Nk^eTk&xorEzpnRrgK+SGn>nuqWL#a8u9^?g!&lQ?1c}YVwm=h?flf{|E zTNBfIKRsWXg_{JtyJCFhpjyZzTvPf2p%w5Nd}~8)-c!xXdpF+Ko#fo^O6=}-@w2DeW+Xatp?L4M%VNr?q##_Mq-+2MJyv!QU6BII*{veDlpo zYmXmXh|IdlN$am%QZ>D3WP_W(C)O7DWMzY< z+ZC6+| z^J-xBw1PNxSwSK=t}grI@f0FqIx?q|)*dW@daD7oHtBBr$2q+98JOS? zO(%?%ol-dZgNfN4D!G2Zu_6%Q`X0yH2dw=Z1Bvasc53hAKPSN)+PNbG(aghm2P01YRYh!V*RbC*VME1Lr6_bc)*Ox7x5o8_&6tz5gasx@FUV7 z>vgO#BxI8_7tHbfs<%qn(AcS6-WA2JKXoJ~4`7lzXXpB9Ho+CaHfhGJekHbc` zk49si@zXIW)_g(Ma!Oe%z~A>C*P~dVU{)2WypQxa4J1%3!*n`8@5m1_~MeYSsbsUWn3 zJA-y)e6qthlHtl#3(x0Cv_qt3BmBs3X}Af#C;v+JS-s{b0QQ56-fS09SlG!~$KSaz z1t<0f5;WmJcJE9)X!^1=#2*UL+z&9xwvq@eqypV@Z1dYzcBPIKobV2S_%k^_x@boK zyg!<-qG3SqJl8vtJCYbq4HHf!3v#~f&-;^wVc-v!X{mg9oF}1pIqr!cDIt5~gnb7Z zKz2=)L*5l^pSUNGxmw}}@k!nw_P9?3G0#V?h9uZxk-eGOG?;%!y8Neprhc@d*gC{q z0b(l9MTehyK}Lncpph8kzZ5gw$6iWnb`sF13?%K1~44s$^(xrr}+uIQX* zgY-;wo7+UW0K-=w+RI#xD;LgKdn*_7t-#E2gv>ChDs>q!r-y#k0lq1`0e%?U@dH_b zyibaY9BZOVujNmTS1Re8@-!*#oB+IIvuiuC@=tC`ttCnpK=hZ52e{n(^I+h|JUj%- z6)3rw2lNsOE-&_lQ$I)??8dNR1$9b7&_d(PD}mC+XIl((QCo;tE|Xuy zWy);EWt%GFf~FtAC4Z6z9>OH~I=l+j>YK+C@ljV@#!UB&WqMN(6>C`j;O*nuyIJ#` zfk~{iiNcI5w{Ianv@O`ma_FMfS1^LVP_aVjsJu5J84F5d%!pl?kw zp9gS0;fKKu$^4W2;E35xR3Op?aOU&)MP3>Ze+frfBrcO3$Az*5XHNJVVX98>O;efc zzylpE^Yx;Fz3KrCAr(30Qi!SUW1IG^N48wu>1S;4Gxzt!+Y(ncP~DCBq(8QPHmCDN zOa4$|{howzDo!nYPdkr@aDWka-yn>3fS;41ud2;Uxs}nC`&`DS;8?pE9ggM}ejA9N zft}B%%b8^wm(Yie2Bz|{#vq4KEi-1I`Bo)Q~zB3%sk|jGSvI{J* zkVMYMn>TO9xzUv=v4TIoK7M)HK3(0=gEn}4zPew-BkY~(YS11ObXJ>y4uXYH(;M5V ze|rQUTwjFVA*}DO-AB@%mzQIg9c_Afvl)T`8!HM`E%9Q(@ayhXf7*KIo=_ zLiHZ002wT*#t!$|fbB!qhoH?K8mgm*%DQ-NG{u(W&1-<5+hk+Q&s)AkxRRejHH~1& z7oMwhs*k}=kC10_lC5aJ6*pVrk@hu~bk-qJ!+7{*bK;!ubLdO|F-ggVAO~z3-FfnO zdwq9zcfE>MEND~ef}snw3Xz*GXiFee6KLpGtGy>g78OdlTqw{cS6}jhwfsNU?JtS% zm=4Dt@m4S;w2nn+Rq*3c@e?mlJK6ZbcUb636QJaG*H?D{&w;}?R%(W>)h04-qz9YG zHTs%S1}y+2`$f~2D9(%CIW#$?O{DV!So-7!?W3hj)sPyJaeP%pXbxaaA?czj>PV#V z_L3z6J|(-b{~r8;2CV&V0X>ymm1QP7r8?Gq1fTwS-RefUvs@#XpX`7y=r*7u-uqrT zMIuK)2g_y*e}&F*^h*?@Z!VVc0adxl=uf_qRk8CLqLTzU;U>`I@T2?WM zPdQA(giT4bOl64?SPmbUD$*24MPJ6)ssq8rmEOgGQk>koghza2Ku+V7mdq4sk%ma7 z2-{Fy=PVJCMelk}lDnIao&e*W{@=g*r-R}sL0=IA)l#7-O;ea9~%G*QkH!KG&?>Oi%Z zcVm?zH7!Q{Om5UsMEP)%PZBb*rF`n;d56Tz#p?Uh{Jm7I!DFkxk@k-cpWMFH;2(?s z$Ur%r#aL7`d{)p>Z)Y9^;iR8~M1 z&NS<4m6z)@EoRpxecEBpHcR^YKSz4f{^?-PluIqpqW5Zl1fA7b+9+}ObniY7hA#bV~)WHj`K2!I+uaA*5(`c$}3~ zO^=%}5Iu)qVJVk3;?UjW=ENqnsk94GN~PVC#gYN65*yhj&9=&a@7M%_366upVQN z@%pvHf~cqCZ5W}=F6@J>p5kwz3q^6#+wXkuyc|}Q39jL!`F$F=gW1@V$Xf zSv4|fOXv*D4wOO}E>p*C&Q7!fUg&aSoL31eyhf<>!`%>M{0LIJn9`|k6cxOBcL|;z zU;=LVA=Oz)|3YV?;iC*ch1}D&LYKf+VDCa{RZ$z?z|L#%_%#Z4uXG)L4!%Dnvrm(3 z6h8;aYBRf=WX(04-4C+0^L@AGP}f)m?GBA0K8(-a0vDf*?zs!JPliw07sDr$55~u(lR6WJ znpNQBR+Gs`j=6;Y^DaEFhkpIx25!=7~qmxgr=UwoEDE135yf;VfSnmA+fCeFZ zu}FBFjZ?vH(?AeCXTM^k2r0FkwrVLzL?CehrBdkykhocEdmK;W?3&%#27~x_X4kkP zL@iwG^=RIkH}9?I)R-C@Sik+aF4k%;XEHc_s`?M-~)}_jdh^A4uVJLb0`DD#@ag7xh1pl z5x#Fx1`jsCUK<9sa2Gto7O@9*y4r$mKto4Tl+y^zL7O_FTYWFeN8VAWjq|`7>BLfGVZx*-x&b{cp_gIncVoJ8sY+dAt z@x&Fe1~lvlJxC?qjO2ZkdZs+PP;u%jk`+bl9pHzW1>d|; z(MPywT`qnJ_OVf!c1{l?A%2tP42)wsYqy~Eh3yfMkgO0aOx7 zIxXMCIN&v$uO2GK9e$@Uq;&iF=0-A6YL2F+M*RY=V}#7QssVVMtycSQ+cpsXIsPjS z4N%xZ5_?#J4NIHCi#v#3HgY{?QP#cw%3Bn5{)f`XeFE(VaR9R$gew@rNcEu>|Ba1g6 ztSe?^${1@|fs5P)+;-G}2!29H(4lfLlNJ-m4Y)ainvVK#kA>AF&?zj+%;{rSNMlnh zKwGvgt(;{DnoYC?y;x?rK;xuiD`^s_((}vqaWTs?kV@r6qRn0Qc)}nPZ)(Bh4s5%=3OQTxNpvx+) zHSnIaATs#DZ6!~XLJYw9*710nVL3zTKT$YX=GFu&YP3sA6Dm~tmr47f(2mltVzPW@ z$qg0;D*KexL~vCFyL81-y^NGGy{*2@0vEF*2#O&d%4-MGJFPKQZYY@l71@+o_Jz1y ztudE9(p2Fkj+TO=ol;%s!Lq`kR z{7n?Q6KU{-#EcgIt2gG7Jp*7nh-bK$Ng}*ry$KuV=E&Ew8S#33o?1Kh*5PumlbVq> zmHo;099?A?Mm3*s{@;NPJWQ0= zUsXLC(7OdC+403{FdTvZ-k2S&t)2IZDLOT$<+LIszqb+J@Qz^BUMcsMr=8G`d?^SQ zQO@r#(ISl8xg~gyM(fX;Ki`pd8~Sh&1X z|6@r#=I7t70(sq%Y{S@Ylkwgf+3u{*bm)VT9%G)M1}Rk&IirgG16;sA$_xd5*O?;K zx`8Yie}TC5FClWtKE4-toOO=R3W7io#_vAGf)|zEpj#1C1ReCgY&vUGIUBR1;oE1X zgp#+JkMGYfL`wx&bvmS(K5ve@ZIs-`VQAxc18*rB!+2g-}_;&YeEuW@PbrM8TuuabS?9-p?E~^YD1s`ZSV#cIET`(6L_3ukHHRt zFbsyz?kSpZQ37)C=$YsNFTTJsMkh1ebCa zAJDdx&Lb#I2nPKN&rwz}mT6@>RM0g1Q9o}rzq2g%+Tt%_+6Exvi}PqqJ1T{q2gp4R zBLyBo!48m0Iy(fAz3`B~Md(*a1`TfL=eke8@BH<)jq`f%wsQ={)+Zxj8 zllEa0#Mk80HGYn^(~5lo_L5&*xg>a;g^|H(!!Qhn&(Twu=F)|*^C0K49SVc7yK!xK z(MaTAJMCcX-KR7yG;POxF_!;N`X!a**`Zh4*Ju6yw%O%DSIz-B-u{YG|7LCK0?p9b@W7A^|MNz2fEKH|y z_lZ_o{No?Lax~{sm#S$<6(c(Rw3E$Qu{DTOD3N6^@C$(j$aA#~c${s`!3x4K3ib2I|T)A`9t#k9R>Cn^f+9u<8ZTQ)k!4AuZ<$E zx*&L)fMSAtSZmo_)fwt+Bdt{%+4kBkEVG-TXcH(RL&E-@O~Jfan5KgF zU^(;>+9ub6@!#8Oh#+uVzxVF-U5vG^pysE%*yp0icbDRPyyoJ-VvkY7jKWYmu7K_! z2xLnZJ4mdfRyr-nQ3p5Ca`^0zaUve`J%p!cQT0M;3@ z3xB2rwr$*MJFM;q!1lGn!*_$S3K1jtaZhN7Hag&iGZs!5g$#O)_><6m3yPYI5(<=| zSJD}*f;R?TVa?w7tBw=Ag8_J)rB+Q(+dvRK<6kinm#~VG0*6ZE1Qk#!Br2Li5kl6+ zUdLPP-Dr1Bl2-ip&a6LT8wVZ|2RLoz0<4R4fv>yZCnbWgdPF4)L2*VrK}f zGezV;FMf&0g_*Pt{saddD=_IqKI>UaU#7&y&4SliAIXTs5>lb8Tq+SmqzmQxAi&V? z@#^;SXLx>pfBx_^gDHbt8v$%OfXa>G+YYQ$4wRYoEIi9}^rm7T9SY@)$JoR2Akqq% zMJYE(kLQAH4SKF)GQ0vS!)!^uZtVQw`(=1>Gyeeqr;YXM_WZiCo_r31DGPwXJQa0n zw_OR47OX?&!~jlVA)TEB)P`=EZSpH_EI!WQxZQwaz~(cY?aFJvOESK+__C?8{nEF= zru_Kvickn$iDZ;ynxQgo3Mv113!HuF|o`F``7TB9^vrOyfVpT(w<#wCHs5`B~ zsVrcT5!{In9xO7N| zCDwa#2vy|uzVL^p)TZ&xG%~XCccWJcS?b6tmhTC;mM(>rMn5N!$`-^X64pX4aP4i& z&4noOpn}yj7V25)wb~c${^Pu?oUK5JcK zfdP1&byeGL+c*$?j=o~xyx4-1*p{0-7=7qw(>7>P7jCv#V0R(VB4x9cNQ0yj8*Tr+ zGo-9boCSg;9?qOObLQf6nJX@G`26PfL;5~C$N!}g$B)suR9R7R0Rz4MCNg)q9YoPp z96=19S@vCU2zJ;C#)U!qRUhNa{ku1Jd0Z$Z-xfS<8 zg!9OXN`Nd1v0();d|_*i$Og-oYJ+}6wh))-9LQrb{2_bBtbn!1ScQMdpsI3>IkA?| zVf0kMyq<(X@flM}I}rO!;2=!L^JF@{zMhOHlL-+QSnJ_9AAznSN9Ldq%sPTdTFE$b z%#)_<8e^&OW4dR#=I~Db;T7|BH<}_0Mc3+697jc*4w!jv1$9G~(jf?%lNk!?EC~X) zNXirk@7i>rPH018O176}RUY;G>g&U@`i;Q`VAHIIOqV-mM6Ejw$-FK%Fd_^k_N^He zZt1uE^OM0~ptQOuHOios%!1c|FoZGMfDE-;k!i(iNR-`hYOyU*&EpJq_)vJ`cnM)7 zLa)|H+|WFQfSu7hZBe$o^1}G^6ds108ZP<1-KiF3gY(zSKl#-Cx;(uW_kaG~yBK+X zGoR3nfarE>qonXvs$tPC82>al5?T39#XzU{Vs}dK@2MeGO!ehdJChV zS!{nGh!yf+RpHv@|6flWBsBOjTD*J>LkBxIv3rdO;)RRUQ*ZI?p&`|to?EODhki%5 zquEjjiAev-Hwy>j$>6yAto7oa(xJiR64blyNNf+ z3deT6>p+C~?>DpSm$hT=Xrw?g-g)-fd3kp7N-UFzFX3hsO#N_jGrO9Mr{U$`6&gjt z_v{0n6dTA`E_s+{Dpb3S!{gxKWC$|j3&CO_lZ4^P;J~tlr z$?c7&ArMidiG+Kh){vyY@>Q|niE6=}U0Kd&lgYI|8he1=;nnzK0QYGTN3dpF4s*^E zVE9ido#%v`ihIOpNPriQ>o{y%N^oGCSt!F=-&L%hEXcmqU*=&GI-t}UeQ}k_t zZ%hU3fsZHJY!F-av2X0Zx9pc!=jZ;I?0*;rQsu=$mGL_rP&H72i9}hv7ISBT(N)k| z4MPu%d9h|W3^DAUb}SOfm5bmf!vPH82Z8>0|DHYrtkXD>pw?Vh9;8eC*&uqbOAFW( z2&pTACD0Tl#0>2sp*6X|VtLMXuko|qMy}Ibg;Y27W&};-*n&z*rBFyjl~`qZadi;# z@!;s&n_Xx}qz zkk{~u^9+Qla~Nz$y3g=R0<^6gnG^6tQ8OwPBLS7Ws1t8ctm{e*GyR% zhA8FowekB*c~1BiGlQqoL2~%Rf@cbrX>MiSEgftHqD)G2pfYH*n+~JiZk4A+hJ;we z^P0A9teW?R$vX;9fB%(xc3bd0NVcPm%VX8Cts|SDs5(h%4p9o8bh~J}aob=`ZWO!2 z!&!>zX|PI{p#>s>4yYE|x#G<%X+rm+BZ( z%8EXtP!Nkd4ZhPoNh?pe z!%Ts52 zdHNtCRBYh1FniZG2;aa+nn&1*4!TRrE0sJ4EO`m zNZOOB=1CS@xM)wKq-k4k(04iMSMNYuLjJFt*`oZSuS5j;`rVAbS}ke(Eg6p|fj_$q z#-R0AuMX)tst?eEIjlhcgUT-)$PDAh^a7b~XzW&8UqDS7_0wnK9J2X5=i3+l5YS=3 zE&GvLb^R-Ftqusk_`%s^>N{JOyU(^|KIMArjbF#@InR5B>>*+i@SOgpi_qcbpfA#R@GzmJZP9>@4uVlsNr8jjelV z@?yaev{sTsnlAoKK0qW^Lh0WSk&yV~Unu4 zOYwm3nA6D7D(o!B(>=5^OAB>kc@l}`;9m_?68*X!c$}S)F$=;l5QS&+D-OD75Gt;j zLy0}CtIJB4g9O?w( z2waZAU@d#P1mbqoYS^A zpWjxzoDU3g48KKXCj1#C66E>~A(ijn2Qn3Rd$y>`vN(90l~X-y!!Qt?qhE30#W9A^ zHJ-Yq(4o+D7Q&WmQAHBcskD^-_nz!Db?lfl%Yp9Wy?5{IOtq$wEj-=tcl`0;dCzw& zql7Z@EXM-+SR*)r9oH%#S11qg#a4AEJQOKkvDLd(4HOZTuGn`q8)DRd8K_7x5JEIT zy&BnoG6=?(Ahfpiv_K$DbUcBC?Oo5sdT!^b%unEXZB0NJG@|^#5{ArF1#Pgdju0nh zwU)x?^VSs0UPtSt)(z`jY|#>yhNA;jo2b!xF$BStmlT zJ6{qwu#Oi>#r53fqobp@3DTc?k&5L4$$ucVwQteg0(s3Iy6@DWt15(Z(locI=r5dz zM&HO{b2`wkEPmlqH{6J2BviFPCX9ZowB!$-EvH zW;HbBG`2+t^4;A(sfKZrG`J-s0XNzRJ7MC(nhcV%Cwr%Z%VLk@}0p`4J)AtjsYGy=&IUQ7Kpx-{-JY_9rOgTXVY zA3;5H?zkU#oQ0504uUWcM%VTfldw_34Oko9Y2*M6p;Ksr)21_+czYWRNYucr`t!Zl znP$$CK?~R0<#an%l|HL1?H%FrNU_AB5eU(@5t&P+9^M%+jzeERwRF)BZ5IaFe-vi>Ik_w>2bkN4Qx2H!PMHe zsZhIA@UHnPU~G^?OHy^dHw$X?ruWAmWbt8T{CWPrd41=0g(+a!e99(?%8-n+>IDQ` zwmZ3t0eGC9RoiYGHxPZczG4z1uq7x?``Q$JOOOCYiv-DAfCaT2Y6)}6HOaNK{ra5Y zuHuy}q;UmH=4CEt&K#1L)_Bc^K7IY?%e(5s;t~_<`Nw>5X?^W_&Gcu=8k;xGU&m`U z$kO7?iaxOGC2s_h)F$r>6$Pb!K{5}y)0a_U`{^3#wqx7SNj38_nAaoe}-^B+^Z z)I}gR*LADc^wNZ|QITFH{JmULx~~d?Yk2*(Jf8o)Tuvc&p?8`#OsW%pqg9!2h(4N; z+5-fXxeWy(+M9okkH3ga~3CxQyi=h1F$eyy!BuXAMnRnTweapD+A zkUVj3LdtV@n2z2O;BVVno9OxeoMB< za%Vw&Dev97VJjY44?W?%AUE*cWS_xhkVoKXPweF?l(QG~-fcTsf-2O6BsT8WdO`x*^GPs=s1 z1{ZmvJ{1Xly}gwaEHqi)VN(a#`;f_na7WJG@p0H$Pb9#ULXX5w^-U$;SDGi;mW3I6 zU>c~$`s*s?!ST;281KpxTG;FEk=2Qw2dW}233+bEOWi5I?bQ}~f0S;aRi5W(T?$9eLvDiq-|o47Uh2gqPJuOx?~>c| zD8C1IoKwzB%S%a3QwVZ(boOv{bn6+GD zT*|3=DVb?p0D9*t?!Om!oSje03c@fDJU3sl(2IutfhP|>z=Gh#V+hTrUD|FWiPq1z ztrn#R1urw~ursrpl4XMyPMd9h+^O?AKd8M(dx;ols=#|h&v0{OfS984Xd73mPB--; zM1L(Jd`ODeIw@hG;=5_Q z)9Mj6vrB_=|<2V$4 zroZBdc^Ig=P}oPFP@7p01$N8Kj#jHxWRjEC!nG^g;WDH7?|Y7&HfcgT>aK(!cFyHH zx9@}xvd}peFik!^q*uKU^ei>5jy>mXnR&2Sq{?JZ@V$4C&4qWQvP?c_{j4nfPPJauz%_V@YlA9^YEN+GXKg|Qi?FIIZ8`QxJ31ejhw zJp6S_4=_Ay-T!_$x=F?`Jn!{R`#tExNGRpN&k@`bGg(MNP?S3J(r9)9D6%<+2?1-V zy;LAMa9>*OKpXuHtr@T_q4V`UTYMI(M0#=B^KVP!WHNF#81hl*qs(aMVVyY6&;a7) zW)i8PWR&b3Ryhx?s5kX1kDR+-=n;{<4JGwwB)fq57tvvvjbuCPU|d-pBH13I!QuuWZ$kSs6@Z)J&M4z`a&)fHf+S=L{ygZUG_( zQ7vF+D>ZxqAz&t-Q3n}Dw5fd(x~s}vNDH~Dk^T4nsw)z#b|U6mE3R|u&}izm>p!*4TEKM)2V zIkt8xwxnUjJ1v;hEe!C8d*tI)aykA8Jkp`M$0uBaCe7U4j}!WPFnlHc*@=z}$F8QT zmP?vs+!NMnkdbjID*Tz8k@& zQ#P2H#e(>e4RXp@C^}u&wB;bk|J=hAsRv!0-0;?mCz^}L(h~=J;7#CBZ*)Xx-BPdm zV7L9RyhTY}2)y~mt)_vh-b=V?z{M%Xsc&q{BKqG4ry^KJ7?)?;|I zNRh~vqIpKk@o=1|y_M*;txXb1Nrbm&dnO{n)~)WQw}m`_YfG$mOnXgM{=4by0)%>5 z)q8s1?~YLV|MBUuW!sb4ZA*1UWuvQPbdxfneY&6CUVcvRZ(z9nZ8DwSPvPj7v*R5W z?vhFJXEGtN^BrPeuP5Wt<#bG9zaDRp#T+Y6r&t!PruM?l(60u^B5ykgrKw16*jy)_ zNn!EM{vw`jkZ{#e_|mzfbT)?8qBH6_(udTbF>Le0nedz|Jy|N%IvJ7f10L=a>JS|ni~7mM_F^EqT<}*BwO~p=Xf1q)+lSw%Y+T&| zjIWdR^B-TXlh4%&e#AM6E3AV$ntUI3@8IC4q}bhk)ZVPw_#A!*yV3MJG&Jg57C?Lx z|9jzB{{=qbT#d8C0eGB^R!wi)I1oMCzhZzau(sf)$sX73p(v8=0!;(By$g&qw1`Ng zKvH&%MgRNGkd!US_HJ@WqWOF?^El*(W)JJyKKn|IQX z*FI>#XsEEW%6V15g4!;++Ao7c2qo&hIJh&vVDQJlV4Upz5GxB4c=xK>JSEAywiF(d zrcu@!J8hW70@V&%QM|6lny#&lD5K(7y)+gI8gQZsS63erp6FJ2-{!3~KM(8qul1Bx zS^S%C(J!X_T9UOtw8%VVz$n5Pg1yv>i-A_KGpZ!&ShXYn&=O`4vGsCTUgQ@Yn@&5m)mSo+0^v`%Uk`+>@E)z6iW@<5RC*ibeJ`D53djFT z=VL?hX9y6nOzBNbIFWw<4zj>w)-8@nx#>0Pcn!8Z>dUsPFn(DRtJ#gHN6dS)NObcR0&1@;0_OsusilaPh$w-IX-MgA z=gh}2WJzA3A&V<`EIL!~uqTK+G~A*yKtlm1KH=JsH6xRM>x_UDA5~qBJ2j zOc2&&Tq>wY`PPt%E*!Gt&!FmMDN&!42u-WsaflbB6@(QfU4_hhRo5g0hyD$bBD@d` zge3i5*s@30DCejbxp`%Xo$IK<2auw6v?gCV5BrK*yhjjJmMBYJZ7){nqPqQ!*UoK& zc8eTId5-W*abosp#WHPZZhkl&I#~@>6weTkm9CM*YC0wtmq|F)+~w7kjoH&=J|k9* z2^Yu)@$T%)ovuDcaLV#F2;(a+tv9+@xzcUmW`E&W6s0@zh+zKifR0L-CQUJ_4E>_F zh{C#br`Gg0p3-IRE#2aT|JuEFr6+@0(^M|L3McOgdFic~&?JY8FwUNGfg)!v31Zat zPn&3IN1y58_Uo*OEdIUv2H_CNgx>=(8W~D;Y~VOjdUzzCXdm|6X`S~DkxqKdgvbJT z_ zvNBaxCSJigwL?r&dLgayj+{@~4KC~`xFCLlL&~gKQX#C={`hpq`^hNz zOP-rVc$t!L2#u#F*RLHCC)-CRzO+cfFC;x7ym-iEWh`|%s6&Ml>DawmXH@^#&`Oau z4Cz58W(kW>+AnoYo5&l48VT(AStr0JE>z!)j&u&EekcP(?NvC|PUf&o0Ftz1GH3|v zV^}SPPTR+{EV=Qyak5U169tx6Ls#32%)?Z9fQ~?BS>%$z=45#qINcs+#t#9z4?8XU z4I_aACp}t*8Pi(kED&6`S4o*;?5>8dw@+*^yBc=i0Xv(l!Ft7H4<`z}3fXGo$(2Q6 zEGpfC(S?dsfQOW;a6=cKcVX?4TuF^-2l7tV9)Laq80!K+y{M(obEOB>4f3h)pfqT% zB=%@WQr&l?=1ncNQ;*yJUsS11XAo&>3 zJe<^}hC0@^RhZasduH0+*a{QzE0}Nm4hRnUhx6(EJf4RzAZPltn{d2}P2t;n)`OQ&LDi#?TYN)XrvDO7OJ?mtFGEM37XX377zE&zQAV$6@J;DedV%20D*qlpe zUt`bViX6I0UOE;&NdF|IW;a{zSv6F~`L7d25G{G` zg#{^@O;Rr!>?J|lP$$!qz#%6}Ps%S)2%|1-hIcdyW7$^7s*UxM1w{DMq5d@Bb=!J_ zW@$(;5G0UhYJ<|tYw(Pasw)#tvc$xmqw3p1he(#kD@YPg^vKoElB2J}B_vA)85s7+aq?O{xkNwj1Sk!llVlZ>dE`cuA`)`WIX-Xuq-* zc${sI!D_=W42I9qQy6mDLN2{Fr?L$Sg)-O!7|&K3vsNyWT_~mRUONpXf$1RQPyYS? zs(numd-e0R|J2``kDcB^oq;D@Fp$W}MkPp2pN;*~h7Y1{P}x8dl^qPLhS8$GV3~JN zB@%xibn1fq8wA&@7OHCe!5ojX+sQKreu6%*MV-#h#7U3@bsRj6wol~vsyWem)xo_3 z@H7kd_J7i*^sH55%|kns*s4jzcnT5ACgps+FAl_dK;xucsGE_!=M;4|^gTQP$w6%7 zybXAqUCloW!axiK@VWOX0=H=C(%H$apin`aL+#~ku;%10iuKc5vG{Mx`|)_O?8T9X zZM7_MUDvDK1}kGnVxd1ZbW@4~EDEV>N+i)Z;5npO8zd=Q13SpI+4oeKEXx2A-w73{ zrxQN!Up~@+!TUry`9%^RgdFst_=JwXg2bu&dinOYYa-<1;(TE&iDPdbdr(GJx*B+# zb&tVr!Y~Ym&-GJ8^fC}k+OB;7I7~tsyGNl(S4)i(r3nln-kn5*s-o+~vj2Sf{kz15 zq-xk>Vj-36)G!V2GRI_&*>y8!6D3$|c${U7QES356oudOD=ze9jtaHVz3eHonaZjuS%tlY*xX`}#*su3 zfBjm@YQeXA&iU>anZ{~t;Cqv^54y|cnnYbT7N^y7R|;P7B`+isf(w|Z&xl}KUa@~* z@sQ-~lkSU3mUQ{WD!HXUut<_*+zTpJY`VT>S(*@HYygM#ReGcGpsaqW&(;U1kIKPo zi!tJsvY9_5{3hvo4ph7%^hU`k5)fC{od$->x&qsX03vXVf?%g+^?!n7-Xw% zot8wM&otx@f9+{%w2lIJoQ+s*Z`(Ey{#^fxTk;`x;<#z{EonP+TZRVPnxV;nVqM|U z5@ivUNR6cIdPV>Ho+Bk&w&k=!QBx%E?z!h)Naa{%I+YpSKfL|%&tImm2gi6*T5cu= zPfrJQO84H*>g>c(A~ek;)lQPqR0_SKrC7Nu5{?ATR$h{@R;Rli$MQnjtT0P*B9qu=m7co^);H!H<F#lAvHks;rFlSYTrx zrK|!hrPHfM)w2mCB%Qy6LMmlBla>^)77QB>6opC7Wjdx@>LdW!bu@)))`VwPt0JYu zlvN==wVgLrN|<9KuTK!uz`0{9bC ztvYBcqqGNH0jQHk>YoU3no2K{{Df7ofT^SvUKt(fiK`(Z0UKr3l%&9(%VkkCx|X>r zWFQT01T|rm7NSroEv*j{8|qu|mj@{Agg?UI@ZqNb10LH8sp*WKg>A0E5cn5frW1OA zPWcIG67@i1Lw=h?bn^Gb$vB8CTf_lGqpN`eff?_PZv1o}!!)6rkT8)y%4(eizNB_F zl>1z@kk$?}gvv@M?E?5Vk$Ypjj1H=>=k8awc~li*bwo?$^MI^jK2=sG9`RQhtxQcz zV?RQ!Q5CcxyjL6TO@QatEQfA3ru)0!Z>IORf8C6(n;m#2lL@_+R*r)yIBwutWZrpU zJ<|itZWj0fXF4U5ArgA-^KCVaJ+Tm~fWc9$j)K8j7bO`tS6suC@ZX@J4f`7YWZOR{ zJ)-}-#V7mF1Ah>e>TjL zGYprT5OZnjdCrkiF?nMurABYM9ip5_<|R#=9Bi{sIma`v{hx$`WsJvxFf>GSLq0UcVw=@Dg;KV&nDT%k^OQ^tyd>`#|)WF2?+C+Z27_qwSoZ)>ff#5GAVwp$hYG zpn)!ILMuE&l?Rp2Uh7GUn$SPk5zun zZGw{++&a)8b_3Iw+BjOv!-~E_mXNarN+C`6 zD}jAvG)bGbTYNtoby#fJJv(DAll_G*#r+K;#O+W4R|ba%D8`YsU7UxSM~R#AjW?v4llGb}g)9EKYvG=P= z^7mXbJ3N)2DqP^EnFe|EHEzPyCO1VYZF_QrUhwHaZbs!_)0!Du3LHHhD3s}a1A{4_ zVph#+FQ?PtFv<;a@6bLQUyi>YjYbD}>W#-1h7tg9RqZ6;_Q8_jiq(^Z?XmgG?Qb`s zKcTq7T@5{)B`Cvr6fViQ81l6nm(DQHA1^xG8#i_4woUe1$v84@9n27J$)<;DfXZIpRGTYHKftN>(g`hBC1?k!nqTb zCFPPv*!cY$Dt;I?Lf|K3AW)tngBoOjnq}YdVZv63$RR0)tiVa!&E45OAMLGN>^fxclP%Ua14p};8F$(X6B`0h;yY@l%y8rDP-oADCFnm z0F}fi7nBxjl;vlpXj*e|DW~S8WTq+TYoY5@(9-7u0AB|oS+$MV(}5MSk9FXR(XP{eRA3NSrnC~qu8Z^bi_VM0a&B3<)wT!D(&I|~nqNC`jz~xoWjW&`GWyIZoqhYca;u?Lpa`~yc{LxF`R5tUnIZMWO@sqHyQl6PLfdqUpMLR zFKsHsi9)EImeG6UZ5D;%TuRWS~w)zaYG#_ z60Z?%64!!Wj)ue0&&lBX9dvV5Rch}1JbXmn^LdN6T`jPoSr3zzhc8?!P^i{(VfYb@ zHPQk$z#+^qLt2DsvKdxx?FH~)N5qq}qm<=VEIE2R9j<5SwFJEyz%k=?VkD00|A!qv zvhvEdr>!Hp<1W0***ANq-@3D~xC;KdFYy}bs5o`Az9W&7A7sqwF^c=d-!@@#)Duo`6B99XhM>3Kex-I+PJ0m%czd~Ry zaxycTdGC#8)~eDaR`BcR%S-Y5+mG+XYqBPT(m1ju&f8Xc(Bg~*YH#1*U$T-$I}fdP z>Y%X%c@!2hcNAZCS6?(W!XL@h2P*C1Y)afE@4G&=w}?fFPTn$Kj4C19w^fCf?$0rV zbDG(yzkO!ml_X=$d=}A$sv%8*f4mjar!zbmpE(Q}1xig0qrw?;YBmYskACb{9A)OpNuojdZ!e1y%+LS_)W?#J(cS$+-Yhnm5t^F zM&Z+YMj!&+s8eTUV(waZPS)Sc!{rPI^y(@1YRs7=uWBW| zqre6AprSjg*h+fQBCZY2ctAgV-ixgpuY`Q=?hPy5!z{}C=Z;DZl^l{<{W-lCWW_xt>78vhf;d~JCU;}aN_PJ{)e46ex9ep{_ zk8X5mDXmeLs!IL?thr|jyft{7eN#(sgD?y}lV4%AmlmN`J@+(iNZV~oMR;(+AQgwFq{N$|2lzMS~mA!Wj4Z6jCBM<77?72I0qY} zxYOktUCTr?iu~|U491lI1Ls71@ zUElx1A2dwjo4d3ZGKp_rlBb8)7*@iG1pU#(SNgt zBeK6IuG3#1%fHjcF5-NVkqCkS{4<2#NP|5nIIBV$Cc3b(Tf{M?u$G*a!^H8ZP|!7s z7l>W$(n>B(zjU2+H!0MI#rsC{=FRidl4@tM)<5pQ+)W!I6Ri2)WG5QEksY&{`T2yjUhOAB0#nyMN1;4?# zjl@{KtUAxo1@_jkNTl{+-BK)1k+(h}bh3=-K|4G0KyDb{noz1gXY?QT6!D!))Ht-g zdl3`zhj-Z?ouA&z$^R~7WfMTA;SRiO_Y=^05l0&`EzAjC;A!3{vLE{8=3`9Lh_pnT zDVQE9hz~uLo#2gp=uA6vCPc~CPQbiq@DiLu1jJ|V@lhwsIPP{Oa=E3sB;*c~ah8`$S*Cp$h>SAiw z@{kkUF-F(NV7lk(7!Y@@M~Ykki>cBD|I1a%+`N|Z*6fD}oVlTFBpt_bXLyCNC*1ni z=;Xq`jjO*s|AooBSC@Pfms+B;RI9(aSk$w$B6ysYQ9WzJFc97KD-OIg#t=xCK*mDb zKya3%t5D>#b2L61=}g-G`btXc#Ld)2x{r77J)JVDxxyM=);qo5JumgE%0f_FivjUl`E2!`4u+=X5_?sU#Wa&Q!_qpM&I+=449nL#}OP+f~8+J&sL_0hfTi z3#UFi(}#uV$Z$tlq?bo|p2Ukwazc(|+0Ww3tzJKE%Qtl)aF~3c_7aqu2{~~Aw|O_n z=z7{A9veX`!;{DHTmLGU;!xNbl>5ylAi1Ji{oa9$8&uzvro5ZIM|hl#R84Q&Fbq9M zzk+Bkju52SIj0^rY*?o)f9cxo0L!#0ZMgRLKe{{B+6}#BrBl#ZrBztQcZ?J(M zclX~vKd7%_%aQf?7Z6+PYnKheO~i8`JhV4LB#OBvc+&0{MtBjMORxr{(WH-JV?9A7 z8>G5b5u07LESGy20ZxJ}6~~dGk{ySo@k&$n$(}qKs9ViIYW1hxQtkhnMzz|Z_PKl4 zH}Rg36h*{Y*xE!PKF<*C+$_3ad<+TQydq*97(a>uszZ#qBNZNhVojZ&M@ zz<%PrDz?`8*1#JCHbE*K)M9_*8tc&OGXjO7s?Q9(aM3Q8%%KdnVp1-gltI$gr8ib> zj3pbqX8Z6rJ7w8O-F3)=v44{&w;onz_E&a&YWCc@J$$gZ*lhl>`_hxEQx|*YZuc%< zy*H(DmFRCjQKtr=nycmLW?yyrPm~Y4ybIA5y|E3jozs7%Sn&rZvxeKWFL<1FQqgW2 zF%W!?zhWg`k`qck;HpX&AfakSD{?50ma+G^RSx@PuMeuK{~mMX8lcI8CC_+vyq@)l zDr>NUr?MzlPjW6s36aHjLX61jCK`kS6ffdI9f5g>uDTl&Y zF`X^Pr_}1#v?yoOGj1|37LOO)YEdqW$@dF(@$-IJEGF}@_?rt6*hkH9b~#xF3X&`_ zsU{a!x>g>twO3Zx2k2~IIFcSK^k_Ba$GQ&v+JynBEn4%okCwgCFF0W9cFNlX?lLHA z3<%0&%3|9_Mgt~#vKwf<(-^{G>nh+Hfvlm9OjHd7)J&-v$nK7wsd`JS*3Hns&iNNZ zUZ%yrFSyf&PM?%6z`d-fH+lH>V+L8(UTE6%ZrbO4ga^+J4FSp|t7vn5s_v8@!#UAJ z97E^GK&IC>xxhW6x6t?Z&ngC#{pa}P++(AdI+U&AI^?4Fdp}N1wkgx=|7M(pn-9dv z`xB`cr7=tA6tONb&6tep_Fq6-6JV6&X$ z+*rhy$RAY;;=glF+$u#SLlVDt_nzO`nO4|<;vdi3%|>ir-`8)O$EUS;VHv?1;R?et zjdd3q;LAg!mAV_>x9yTK=WVDx9{~YId*QtFuwu*~2Z%=dUN<~1W)*dhE*KCBZ}O2X zN?y0p#&1BUcL)vddfeTt{x#s=Y;jeRr{5J@Oqv#q)D+nga-q)8jBxsk8h*%5-=J>r+Ln`W9%rD0a82r1>@JEXpwMKmy0)Z#7BX6oB<#J z-~XQypAxW>2o#vi;jlQV2)@M}%`bi&A7qTQDWQ|62zcsc4jgyb0GOF1L<-OAdy2-D tFDyW&#L@M5oLj;&mt})2695Vb0vUd~@0o~~*#qY0wst)_Zp7SMgj;>}a0>tc literal 0 HcmV?d00001 diff --git a/test/test_pack.py b/test/test_pack.py index a0c8464e0..199b6c61c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -13,14 +13,13 @@ class TestPack(TestBase): - def test_pack_index(self): - # read v2 index information - index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') - index = PackIndex(index_file) - + packindexfile_v2 = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') + packindexfile_v1 = fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx') + + def _assert_index_file(self, index, version, size): assert index.packfile_checksum != index.indexfile_checksum - assert index.version == 2 - assert index.size == 30 + assert index.version == version + assert index.size == size # get all data of all objects for oidx in xrange(index.size): @@ -35,4 +34,14 @@ def test_pack_index(self): assert entry[2] == index.crc(oidx) # END for each object index in indexfile + + def test_pack_index(self): + # check version 1 and 2 + index = PackIndex(self.packindexfile_v1) + self._assert_index_file(index, 1, 67) + + index = PackIndex(self.packindexfile_v2) + self._assert_index_file(index, 2, 30) + + From 3b902ed6bf75bb04bdf5703a564f539d8d2e43d8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 23:35:46 +0200 Subject: [PATCH 0014/1790] Initial version of a pack design that should be able to solve the problem nicely streams: added pack specific Info and Stream types, including test --- fun.py | 27 ++++---- pack.py | 147 ++++++++++++++++++++++++++++++++++++++++---- stream.py | 84 ++++++++++++++++++++++++- test/test_pack.py | 54 ++++++++++++---- test/test_stream.py | 28 +++++++++ 5 files changed, 298 insertions(+), 42 deletions(-) diff --git a/fun.py b/fun.py index 883062eba..b2e684472 100644 --- a/fun.py +++ b/fun.py @@ -11,11 +11,17 @@ # INVARIANTS +OFS_DELTA = 6 +REF_DELTA = 7 type_id_to_type_map = { + 0 : "", # EXT 1 1 : "commit", 2 : "tree", 3 : "blob", - 4 : "tag" + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA } # used when dealing with larger streams @@ -42,30 +48,23 @@ def loose_object_header_info(m): type_name, size = hdr[:hdr.find("\0")].split(" ") return type_name, int(size) -def object_header_info(m): - """:return: tuple(type_string, uncompressed_size_in_bytes - :param mmap: mapped memory map. It will be - seeked to the actual start of the object contents, which can be used - to initialize a zlib decompress object. - :note: This routine can only handle new-style objects which are assumably contained - in packs - """ - assert not is_loose_object(m), "Use loose_object_header_info instead" - +def pack_object_header_info(data): + """:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" c = b0 # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size while c & 0x80: - c = ord(m[i]) + c = ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 # END character loop - # finally seek the map to the start of the data stream - m.seek(i) try: return (type_id_to_type_map[type_id], size) except KeyError: diff --git a/pack.py b/pack.py index 2ffc64f52..377963053 100644 --- a/pack.py +++ b/pack.py @@ -1,4 +1,4 @@ -"""Contains PackIndex and PackFile implementations""" +"""Contains PackIndexFile and PackFile implementations""" from util import ( LockedFD, LazyMixin, @@ -6,14 +6,17 @@ unpack_from ) +from fun import ( + pack_object_header_info + ) from struct import ( pack, ) -__all__ = ('PackIndex', 'Pack') +__all__ = ('PackIndexFile', 'PackFile') -class PackIndex(LazyMixin): +class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -26,7 +29,7 @@ class PackIndex(LazyMixin): _sha_list_offset = 8 + 1024 def __init__(self, indexpath): - super(PackIndex, self).__init__() + super(PackIndexFile, self).__init__() self._indexpath = indexpath def _set_cache_(self, attr): @@ -121,9 +124,9 @@ def _initialize(self): self._fanout_table = self._read_fanout((self._version == 2) * 8) if self._version == 2: - self._crc_list_offset = self._sha_list_offset + self.size * 20 - self._pack_offset = self._crc_list_offset + self.size * 4 - self._pack_64_offset = self._pack_offset + self.size * 4 + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base def _read_fanout(self, byte_offset): @@ -139,21 +142,17 @@ def _read_fanout(self, byte_offset): #} END initialization #{ Properties - @property def version(self): return self._version - @property def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - @property def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._data[-40:-20] - @property def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._data[-20:] @@ -186,6 +185,128 @@ def sha_to_index(self, sha): #} END properties -class Pack(LazyMixin): - """A pack is a file written according to the Version 2 for git packs""" +class PackFile(LazyMixin): + """A pack is a file written according to the Version 2 for git packs + As we currently use memory maps, it could be assumed that the maximum size of + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + :note: at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_data', '_size', '_version') + + # offset into our data at which the first object starts + _first_object_offset = 3*4 + 8 + + def __init__(self, packpath): + self._packpath = packpath + + def _set_cache_(self, attr): + if attr == '_data': + ldb = LockedFD(self._packpath) + fd = ldb.open() + self._data = file_contents_ro(fd) + ldb.rollback() + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + else: + # read the header information + type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) + assert type_id == "PACK", "Pack file format is invalid: %r" % type_id + assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version + # END handle header + + def _iter_objects(self, start_offset, as_stream): + """Handle the actual iteration of objects within this pack""" + size = len(self._data) + cur_offset = start_offset or self._first_object_offset + + while cur_offset < size: + type_id, uncomp_size, data_offset = pack_object_header_info(buffer(self._data, cur_offset)) + + # if type_id + # END until we have read everything + + #{ Interface + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._data[-20:] + + #} END interface + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + raise NotImplementedError() + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + raise NotImplementedError() + + #} END Read-Database like Interface + + +class PackFileEntity(object): + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ('_index', '_pack') + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, basename): + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + + def _iter_objects(self, as_stream): + raise NotImplementedError + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + :param sha: 20 byte sha1 + :return: OInfo instance""" + raise NotImplementedError() + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + :param sha: 20 byte sha1 + :return: OStream instance""" + raise NotImplementedError() + + #} END Read-Database like Interface + + #{ Interface + + def info_iter(self): + """:return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """:return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + #} Interface diff --git a/stream.py b/stream.py index 44c7b945a..b30ec1ec9 100644 --- a/stream.py +++ b/stream.py @@ -11,7 +11,11 @@ zlib ) -__all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', +from fun import type_id_to_type_map + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream', 'DecompressMemMapReader', 'FDCompressedSha1Writer') @@ -55,8 +59,42 @@ def type(self): def size(self): return self[2] #} END interface - - + + +class OPackInfo(OInfo): + """As OInfo, but provides a type_id property to retrieve the numerical type id""" + __slots__ = tuple() + + @property + def type(self): + return type_id_to_type_map[self[1]] + + #{ Interface + + @property + def type_id(self): + return self[1] + + #} interface + + +class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the base_offset, being an offset into the pack at which our base + can be found""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, delta_info): + return tuple.__new__(cls, (sha, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + class OStream(OInfo): """Base for object streams retrieved from the database, providing additional information about the stream. @@ -76,6 +114,46 @@ def __init__(self, *args, **kwargs): def read(self, size=-1): return self[3].read(size) + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, delta_info, stream): + return tuple.__new__(cls, (sha, type, size, delta_info, stream)) + + + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] #} END stream reader interface diff --git a/test/test_pack.py b/test/test_pack.py index 199b6c61c..14d8a2496 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -6,23 +6,35 @@ fixture_path ) from gitdb.pack import ( - PackIndex + PackIndexFile, + PackFile ) +from gitdb.util import to_bin_sha import os +#{ Utilities +def bin_sha_from_filename(filename): + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) +#} END utilities + class TestPack(TestBase): - packindexfile_v2 = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') - packindexfile_v1 = fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx') + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + def _assert_index_file(self, index, version, size): - assert index.packfile_checksum != index.indexfile_checksum - assert index.version == version - assert index.size == size + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size # get all data of all objects - for oidx in xrange(index.size): + for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) @@ -34,14 +46,32 @@ def _assert_index_file(self, index, version, size): assert entry[2] == index.crc(oidx) # END for each object index in indexfile + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + def test_pack_index(self): # check version 1 and 2 - index = PackIndex(self.packindexfile_v1) - self._assert_index_file(index, 1, 67) - - index = PackIndex(self.packindexfile_v2) - self._assert_index_file(index, 2, 30) + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + def test_pack_entity(self): + # TODO: + pass + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + pass diff --git a/test/test_stream.py b/test/test_stream.py index 4f022286e..7c9097961 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -39,15 +39,43 @@ def test_streams(self): assert info.type == str_blob_type assert info.size == s + # test pack info + # provides type_id + blob_id = 3 + pinfo = OPackInfo(sha, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + + dpinfo = ODeltaPackInfo(sha, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + + # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream ostream.read(15) stream._assert() assert stream.bytes == 15 ostream.read(20) assert stream.bytes == 20 + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + # derive with own args DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() From 0650892246e99b60448d4a168ea36f84236b97a4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 01:10:46 +0200 Subject: [PATCH 0015/1790] moved all info and stream base classes into new module, base, as well as the respective tests were moved to test_base Adjusted PackStream and PackInfo classes not to contain the sha field anymore streams: DecompressMemMapReader now parses its header on demand if it is not set, using the mose useful 3 lines ever, LazyMixin --- __init__.py | 1 + base.py | 272 +++++++++++++++++++++++++++++++++++++++++++ db/git.py | 2 +- db/loose.py | 9 +- pack.py | 76 +++++++++++- stream.py | 273 ++------------------------------------------ test/db/lib.py | 5 +- test/test_base.py | 90 +++++++++++++++ test/test_stream.py | 78 +------------ 9 files changed, 461 insertions(+), 345 deletions(-) create mode 100644 base.py create mode 100644 test/test_base.py diff --git a/__init__.py b/__init__.py index 8b0e47b19..d79788fc4 100644 --- a/__init__.py +++ b/__init__.py @@ -14,5 +14,6 @@ def _init_externals(): # default imports from db import * +from base import * from stream import * diff --git a/base.py b/base.py new file mode 100644 index 000000000..d12181229 --- /dev/null +++ b/base.py @@ -0,0 +1,272 @@ +"""Module with basic data structures - they are designed to be lightweight and fast""" +from util import ( + to_hex_sha, + to_bin_sha, + zlib + ) + +from fun import type_id_to_type_map + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + +#{ ODB Bases + +class OInfo(tuple): + """Carries information about an object in an ODB, provdiing information + about the sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.sha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def sha(self): + return self[0] + + @property + def type(self): + return self[1] + + @property + def size(self): + return self[2] + #} END interface + + +class OPackInfo(tuple): + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha""" + __slots__ = tuple() + + def __new__(cls, type, size): + return tuple.__new__(cls, (type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def type(self): + return type_id_to_type_map[self[0]] + + @property + def type_id(self): + return self[0] + + @property + def size(self): + return self[1] + + #} END interface + + +class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the base_offset, being an offset into the pack at which our base + can be found""" + __slots__ = tuple() + + def __new__(cls, type, size, delta_info): + return tuple.__new__(cls, (type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[2] + #} END interface + + +class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[2].read(size) + + @property + def stream(self): + return self[2] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, type, size, delta_info, stream): + return tuple.__new__(cls, (type, size, delta_info, stream)) + + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_sha(self, sha): + self[0] = sha + + def _sha(self): + return self[0] + + sha = property(_sha, _set_sha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def sha(self): + return self[0] + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases + diff --git a/db/git.py b/db/git.py index d2477d7b1..0488bc601 100644 --- a/db/git.py +++ b/db/git.py @@ -1,5 +1,5 @@ -from gitdb.stream import ( +from gitdb.base import ( OInfo, OStream ) diff --git a/db/loose.py b/db/loose.py index 37aad8c6f..109782fdc 100644 --- a/db/loose.py +++ b/db/loose.py @@ -13,11 +13,14 @@ from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, - Sha1Writer, - OStream, - OInfo + Sha1Writer ) +from gitdb.base import ( + OStream, + OInfo + ) + from gitdb.util import ( ENOENT, to_hex_sha, diff --git a/pack.py b/pack.py index 377963053..ab394daba 100644 --- a/pack.py +++ b/pack.py @@ -7,8 +7,21 @@ ) from fun import ( - pack_object_header_info + pack_object_header_info, + OFS_DELTA, + REF_DELTA ) + +from base import ( + OPackInfo, + OPackStream, + ODeltaPackInfo, + ODeltaPackStream, + ) +from stream import ( + DecompressMemMapReader, + ) + from struct import ( pack, ) @@ -16,6 +29,60 @@ __all__ = ('PackIndexFile', 'PackFile') + +#{ Utilities + +def pack_object_at(data, as_stream): + """ + :return: info or stream object of the correct type according to the type + of the object, REF_DELTAS will not be resolved in case a stream is desired. + The resulting ODeltaPackStream will have None instead of a stream. + :param data: random accessable data at which the header of an object can be read + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only + :note: a bit redundant, but it needs to be as fast as possible !""" + type_id, uncomp_size, data_offset = pack_object_header_info(data) + + if type_id == OFS_DELTA: + i = 0 + delta_offset = 0 + s = 7 + while c & 0x80: + c = ord(data[i]) + i += 1 + delta_offset += (c & 0x7f) << s + s += 7 + # END character loop + if as_stream: + stream = DecompressMemMapReader(buffer(data, i), False) + return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) + else: + return ODeltaPackInfo(type_id, uncomp_size, delta_offset) + # END handle stream + elif type_id == REF_DELTA: + ref_sha = data[:20] + if as_stream: + stream = DecompressMemMapReader(buffer(data, 20), False) + return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) + else: + return ODeltaPackInfo(type_id, uncomp_size, ref_sha) + # END handle stream + else: + # assume its a base object + if as_stream: + # if no size is given, it will read the header on first access + stream = DecompressMemMapReader(buffer(data, data_offset), False) + return OPackStream(type_id, uncomp_size, stream) + else: + return OPackInfo(type_id, uncomp_size) + # END handle as_stream + # END handle type id + + +#} END utilities + + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -222,13 +289,14 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream): """Handle the actual iteration of objects within this pack""" - size = len(self._data) + data = self._data + size = len(data) cur_offset = start_offset or self._first_object_offset while cur_offset < size: - type_id, uncomp_size, data_offset = pack_object_header_info(buffer(self._data, cur_offset)) + ostream = pack_object_at(buffer(data, cur_offset), True) + # TODO: Decompressor needs to track the size of bytes actually decompressed - # if type_id # END until we have read everything #{ Interface diff --git a/stream.py b/stream.py index b30ec1ec9..f759267a5 100644 --- a/stream.py +++ b/stream.py @@ -3,279 +3,23 @@ import errno from util import ( - to_hex_sha, - to_bin_sha, + LazyMixin, make_sha, write, close, zlib ) -from fun import type_id_to_type_map - -__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream', - 'DecompressMemMapReader', 'FDCompressedSha1Writer') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') # ZLIB configuration # used when compressing objects - 1 to 9 ( slowest ) Z_BEST_SPEED = 1 - -#{ ODB Bases - -class OInfo(tuple): - """Carries information about an object in an ODB, provdiing information - about the sha of the object, the type_string as well as the uncompressed size - in bytes. - - It can be accessed using tuple notation and using attribute access notation:: - - assert dbi[0] == dbi.sha - assert dbi[1] == dbi.type - assert dbi[2] == dbi.size - - The type is designed to be as lighteight as possible.""" - __slots__ = tuple() - - def __new__(cls, sha, type, size): - return tuple.__new__(cls, (sha, type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - @property - def sha(self): - return self[0] - - @property - def type(self): - return self[1] - - @property - def size(self): - return self[2] - #} END interface - - -class OPackInfo(OInfo): - """As OInfo, but provides a type_id property to retrieve the numerical type id""" - __slots__ = tuple() - - @property - def type(self): - return type_id_to_type_map[self[1]] - - #{ Interface - - @property - def type_id(self): - return self[1] - - #} interface - - -class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, - or the base_offset, being an offset into the pack at which our base - can be found""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, delta_info): - return tuple.__new__(cls, (sha, type, size, delta_info)) - - #{ Interface - @property - def delta_info(self): - return self[3] - #} END interface - - -class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional - information about the stream. - Generally, ODB streams are read-only as objects are immutable""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - - def __init__(self, *args, **kwargs): - tuple.__init__(self) - - #{ Stream Reader Interface - - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface - - -class OPackStream(OPackInfo): - """Next to pack object information, a stream outputting an undeltified base object - is provided""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - #{ Stream Reader Interface - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface - - -class ODeltaPackStream(ODeltaPackInfo): - """Provides a stream outputting the uncompressed offset delta information""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, delta_info, stream): - return tuple.__new__(cls, (sha, type, size, delta_info, stream)) - - - #{ Stream Reader Interface - def read(self, size=-1): - return self[4].read(size) - - @property - def stream(self): - return self[4] - #} END stream reader interface - - -class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow - the ODB to record information about the operations outcome right in this instance. - - It provides interfaces for the OStream and a StreamReader to allow the instance - to blend in without prior conversion. - - The only method your content stream must support is 'read'""" - __slots__ = tuple() - - def __new__(cls, type, size, stream, sha=None): - return list.__new__(cls, (sha, type, size, stream, None)) - - def __init__(self, type, size, stream, sha=None): - list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface - - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) - - def _error(self): - """:return: the error that occurred when processing the stream, or None""" - return self[4] - - def _set_error(self, exc): - """Set this input stream to the given exc, may be None to reset the error""" - self[4] = exc - - error = property(_error, _set_error) - - #} END interface - - #{ Stream Reader Interface - - def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on - to our internal stream""" - return self[3].read(size) - - #} END stream reader interface - - #{ interface - - def _set_sha(self, sha): - self[0] = sha - - def _sha(self): - return self[0] - - sha = property(_sha, _set_sha) - - - def _type(self): - return self[1] - - def _set_type(self, type): - self[1] = type - - type = property(_type, _set_type) - - def _size(self): - return self[2] - - def _set_size(self, size): - self[2] = size - - size = property(_size, _set_size) - - def _stream(self): - return self[3] - - def _set_stream(self, stream): - self[3] = stream - - stream = property(_stream, _set_stream) - - #} END odb info interface - - -class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in - the queried database. The exception attribute provides more information about - the cause of the issue""" - __slots__ = tuple() - - def __new__(cls, sha, exc): - return tuple.__new__(cls, (sha, exc)) - - def __init__(self, sha, exc): - tuple.__init__(self, (sha, exc)) - - @property - def sha(self): - return self[0] - - @property - def error(self): - """:return: exception instance explaining the failure""" - return self[1] - - -class InvalidOStream(InvalidOInfo): - """Carries information about an invalid ODB stream""" - __slots__ = tuple() - -#} END ODB Bases - - #{ RO Streams -class DecompressMemMapReader(object): +class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly @@ -296,19 +40,26 @@ class DecompressMemMapReader(object): max_read_size = 512*1024 # currently unused - def __init__(self, m, close_on_deletion, size): + def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no size""" self._m = m self._zip = zlib.decompressobj() self._buf = None # buffer of decompressed bytes self._buflen = 0 # length of bytes in buffer - self._s = size # size of uncompressed data to read in total + if size is not None: + self._s = size # size of uncompressed data to read in total self._br = 0 # num uncompressed bytes read self._cws = 0 # start byte of compression window self._cwe = 0 # end byte of compression window self._close = close_on_deletion # close the memmap on deletion ? + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + def __del__(self): if self._close: self._m.close() diff --git a/test/db/lib.py b/test/db/lib.py index 35823059b..cf752741b 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -6,8 +6,9 @@ TestBase ) -from gitdb.stream import ( - Sha1Writer, +from gitdb.stream import Sha1Writer + +from gitdb.base import ( IStream, OStream, OInfo diff --git a/test/test_base.py b/test/test_base.py new file mode 100644 index 000000000..f7ddebaa2 --- /dev/null +++ b/test/test_base.py @@ -0,0 +1,90 @@ +"""Test for object db""" +from lib import ( + TestBase, + DummyStream, + DeriveTest, + ) + +from gitdb import * +from gitdb.util import ( + NULL_HEX_SHA + ) + +from gitdb.typ import ( + str_blob_type + ) + + +class TestBaseTypes(TestBase): + + def test_streams(self): + # test info + sha = NULL_HEX_SHA + s = 20 + info = OInfo(sha, str_blob_type, s) + assert info.sha == sha + assert info.type == str_blob_type + assert info.size == s + + # test pack info + # provides type_id + blob_id = 3 + pinfo = OPackInfo(blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + + dpinfo = ODeltaPackInfo(blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.sha == None + istream.sha = sha + assert istream.sha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/test/test_stream.py b/test/test_stream.py index 7c9097961..69a93ad0f 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -2,7 +2,6 @@ from lib import ( TestBase, DummyStream, - DeriveTest, Sha1Writer, make_bytes, make_object @@ -18,7 +17,6 @@ str_blob_type ) -from cStringIO import StringIO import tempfile import os @@ -29,78 +27,6 @@ class TestStream(TestBase): """Test stream classes""" data_sizes = (15, 10000, 1000*1024+512) - - def test_streams(self): - # test info - sha = NULL_HEX_SHA - s = 20 - info = OInfo(sha, str_blob_type, s) - assert info.sha == sha - assert info.type == str_blob_type - assert info.size == s - - # test pack info - # provides type_id - blob_id = 3 - pinfo = OPackInfo(sha, blob_id, s) - assert pinfo.type == str_blob_type - assert pinfo.type_id == blob_id - - dpinfo = ODeltaPackInfo(sha, blob_id, s, sha) - assert dpinfo.type == str_blob_type - assert dpinfo.type_id == blob_id - assert dpinfo.delta_info == sha - - - # test ostream - stream = DummyStream() - ostream = OStream(*(info + (stream, ))) - assert ostream.stream is stream - ostream.read(15) - stream._assert() - assert stream.bytes == 15 - ostream.read(20) - assert stream.bytes == 20 - - # test packstream - postream = OPackStream(*(pinfo + (stream, ))) - assert postream.stream is stream - postream.read(10) - stream._assert() - assert stream.bytes == 10 - - # test deltapackstream - dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream - dpostream.read(5) - stream._assert() - assert stream.bytes == 5 - - # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - - # test istream - istream = IStream(str_blob_type, s, stream) - assert istream.sha == None - istream.sha = sha - assert istream.sha == sha - - assert len(istream.binsha) == 20 - assert len(istream.hexsha) == 40 - - assert istream.size == s - istream.size = s * 2 - istream.size == s * 2 - assert istream.type == str_blob_type - istream.type = "something" - assert istream.type == "something" - assert istream.stream is stream - istream.stream = None - assert istream.stream is None - - assert istream.error is None - istream.error = Exception() - assert isinstance(istream.error, Exception) def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): """Make stream tests - the orig_stream is seekable, allowing it to be @@ -145,6 +71,10 @@ def test_decompress_reader(self): type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) assert type == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) else: # here we need content data zdata = zlib.compress(cdata) From bf4437ef45d9115aa2716e1c722c3938b6976803 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 13:37:59 +0200 Subject: [PATCH 0016/1790] DecompressMemMapReader: implemented compressed bytes counting, including test. This is required to properly read packs without the use of an index --- ext/async | 2 +- pack.py | 6 +-- stream.py | 105 +++++++++++++++++++++++++++++--------------- test/test_stream.py | 19 ++++---- 4 files changed, 82 insertions(+), 50 deletions(-) diff --git a/ext/async b/ext/async index af0040b0f..796b5e94f 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit af0040b0f3c6ede3be5b2d6bc69f6ea5ac53c36c +Subproject commit 796b5e94f19dfc36a3fb251468192373c76510b0 diff --git a/pack.py b/pack.py index ab394daba..2c01f0452 100644 --- a/pack.py +++ b/pack.py @@ -54,7 +54,7 @@ def pack_object_at(data, as_stream): s += 7 # END character loop if as_stream: - stream = DecompressMemMapReader(buffer(data, i), False) + stream = DecompressMemMapReader(buffer(data, i), False, uncomp_size) return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) else: return ODeltaPackInfo(type_id, uncomp_size, delta_offset) @@ -62,7 +62,7 @@ def pack_object_at(data, as_stream): elif type_id == REF_DELTA: ref_sha = data[:20] if as_stream: - stream = DecompressMemMapReader(buffer(data, 20), False) + stream = DecompressMemMapReader(buffer(data, 20), False, uncomp_size) return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) else: return ODeltaPackInfo(type_id, uncomp_size, ref_sha) @@ -267,7 +267,7 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 + 8 + _first_object_offset = 3*4 def __init__(self, packpath): self._packpath = packpath diff --git a/stream.py b/stream.py index f759267a5..de7ddcd86 100644 --- a/stream.py +++ b/stream.py @@ -1,6 +1,8 @@ from cStringIO import StringIO import errno +import mmap +import os from util import ( LazyMixin, @@ -13,10 +15,6 @@ __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') -# ZLIB configuration -# used when compressing objects - 1 to 9 ( slowest ) -Z_BEST_SPEED = 1 - #{ RO Streams class DecompressMemMapReader(LazyMixin): @@ -36,7 +34,8 @@ class DecompressMemMapReader(LazyMixin): times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close') + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') max_read_size = 512*1024 # currently unused @@ -52,6 +51,8 @@ def __init__(self, m, close_on_deletion, size=None): self._br = 0 # num uncompressed bytes read self._cws = 0 # start byte of compression window self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info self._close = close_on_deletion # close the memmap on deletion ? def _set_cache_(self, attr): @@ -85,6 +86,8 @@ def _parse_header_info(self): self._buf = StringIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend + self._phi = True + return type, size @classmethod @@ -98,7 +101,55 @@ def new(self, m, close_on_deletion=False): inst = DecompressMemMapReader(m, close_on_deletion, 0) type, size = inst._parse_header_info() return type, size, inst + + def compressed_bytes_read(self): + """:return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we can only hope it works. + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and hasattr(self._zip, 'status') and self._zip.status == zlib.Z_OK: + # manipulate the bytes-read to allow our own read method to coninute + # but keep the window at its current position + self._br = 0 + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + return self._cbr - len(self._zip.unused_data) + def seek(self, offset, whence=os.SEEK_SET): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + def read(self, size=-1): if size < 1: size = self._s - self._br @@ -109,33 +160,8 @@ def read(self, size=-1): if size == 0: return str() # END handle depletion - - # protect from memory peaks - # If he tries to read large chunks, our memory patterns get really bad - # as we end up copying a possibly huge chunk from our memory map right into - # memory. This might not even be possible. Nonetheless, try to dampen the - # effect a bit by reading in chunks, returning a huge string in the end. - # Our performance now depends on StringIO. This way we don't need two large - # buffers in peak times, but only one large one in the end which is - # the return buffer - # NO: We don't do it - if the user thinks its best, he is right. If he - # has trouble, he will start reading in chunks. According to our tests - # its still faster if we read 10 Mb at once instead of chunking it. - - # if size > self.max_read_size: - # sio = StringIO() - # while size: - # read_size = min(self.max_read_size, size) - # data = self.read(read_size) - # sio.write(data) - # size -= len(data) - # if len(data) < read_size: - # break - # # END data loop - # sio.seek(0) - # return sio.getvalue() - # # END handle maxread - # + + # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream @@ -186,8 +212,7 @@ def read(self, size=-1): # if window is too small, make it larger so zip can decompress something - win_size = self._cwe - self._cws - if win_size < 8: + if self._cwe - self._cws < 8: self._cwe = self._cws + 8 # END adjust winsize @@ -196,10 +221,18 @@ def read(self, size=-1): # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) - + dcompdat = self._zip.decompress(indata, size) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) + + print size, self._br, self._cbr, len(indata), self._cws, self._cwe, len(self._zip.unused_data), len(self._zip.unconsumed_tail) + if dat: dcompdat = dat + dcompdat @@ -252,7 +285,7 @@ class FDCompressedSha1Writer(Sha1Writer): def __init__(self, fd): super(FDCompressedSha1Writer, self).__init__() self.fd = fd - self.zip = zlib.compressobj(Z_BEST_SPEED) + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) #{ Stream Interface diff --git a/test/test_stream.py b/test/test_stream.py index 69a93ad0f..41f2b235a 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -49,12 +49,20 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert rest == cdata[-len(rest):] # END handle rest + if isinstance(stream, DecompressMemMapReader): + assert len(stream._m) == stream.compressed_bytes_read() + # END handle special type + rewind_stream(stream) # read everything rdata = stream.read() assert rdata == cdata + if isinstance(stream, DecompressMemMapReader): + assert len(stream._m) == stream.compressed_bytes_read() + # END handle special type + def test_decompress_reader(self): for close_on_deletion in range(2): for with_size in range(2): @@ -82,15 +90,7 @@ def test_decompress_reader(self): assert reader._s == len(cdata) # END get reader - def rewind(r): - r._zip = zlib.decompressobj() - r._br = r._cws = r._cwe = 0 - if with_size: - r._parse_header_info() - # END skip header - # END make rewind func - - self._assert_stream_reader(reader, cdata, rewind) + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) # put in a dummy stream for closing dummy = DummyStream() @@ -99,7 +99,6 @@ def rewind(r): assert not dummy.closed del(reader) assert dummy.closed == close_on_deletion - #zdi# # END for each datasize # END whether size should be used # END whether stream should be closed when deleted From b6db082874b528b431d90de898a3061b2b6d9c36 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 14:01:45 +0200 Subject: [PATCH 0017/1790] DecompressMemMapReader: improved compressed_bytes_read method with alternate route which is a bit less efficient, but works without a custom zlib --- stream.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/stream.py b/stream.py index de7ddcd86..fb58125bf 100644 --- a/stream.py +++ b/stream.py @@ -120,16 +120,26 @@ def compressed_bytes_read(self): # until it is finished, to yield the actual number of compressed bytes # belonging to the decompressed object # We are using a custom zlib module for this, if its not present, - # we can only hope it works. + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + # Only scrub the stream forward if we are officially done with the # bytes we were to have. - if self._br == self._s and hasattr(self._zip, 'status') and self._zip.status == zlib.Z_OK: + if self._br == self._s and not self._zip.unused_data: # manipulate the bytes-read to allow our own read method to coninute # but keep the window at its current position self._br = 0 - while self._zip.status == zlib.Z_OK: - self.read(mmap.PAGESIZE) - # END scrub-loop + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + # reset bytes read, just to be sure self._br = self._s # END handle stream scrubbing @@ -231,8 +241,6 @@ def read(self, size=-1): self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) - print size, self._br, self._cbr, len(indata), self._cws, self._cwe, len(self._zip.unused_data), len(self._zip.unconsumed_tail) - if dat: dcompdat = dat + dcompdat From 4977bc52938c058123f2a3f6e0dbf6fc404550dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 17:36:57 +0200 Subject: [PATCH 0018/1790] implemented direct pack reading - currently not all information is passed on, the absolute offset into the packfile could be interesting to the caller --- fun.py | 4 +-- pack.py | 82 ++++++++++++++++++++++++++++++++++------------- stream.py | 29 ++++++++++++++++- test/test_pack.py | 3 ++ 4 files changed, 92 insertions(+), 26 deletions(-) diff --git a/fun.py b/fun.py index b2e684472..bf223e904 100644 --- a/fun.py +++ b/fun.py @@ -53,7 +53,7 @@ def pack_object_header_info(data): The type_id should be interpreted according to the ``type_id_to_type_map`` map The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" - c = b0 # first byte + c = ord(data[0]) # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size @@ -66,7 +66,7 @@ def pack_object_header_info(data): # END character loop try: - return (type_id_to_type_map[type_id], size) + return (type_id, size, i) except KeyError: # invalid object type - we could try to be smart now and decode part # of the stream to get the info, problem is that we had trouble finding diff --git a/pack.py b/pack.py index 2c01f0452..fa5e43b31 100644 --- a/pack.py +++ b/pack.py @@ -8,6 +8,8 @@ from fun import ( pack_object_header_info, + stream_copy, + chunk_size, OFS_DELTA, REF_DELTA ) @@ -20,6 +22,7 @@ ) from stream import ( DecompressMemMapReader, + NullStream ) from struct import ( @@ -34,50 +37,61 @@ def pack_object_at(data, as_stream): """ - :return: info or stream object of the correct type according to the type - of the object, REF_DELTAS will not be resolved in case a stream is desired. - The resulting ODeltaPackStream will have None instead of a stream. + :return: tuple(num_header_bytes, PackInfo|PackStream) + Tuple of number of additional bytes read from data until the data stream begins + and object of the correct type according to the type of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. :param data: random accessable data at which the header of an object can be read :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only :note: a bit redundant, but it needs to be as fast as possible !""" type_id, uncomp_size, data_offset = pack_object_header_info(data) - + total_offset = None # set later, actual offset until data stream begins + obj = None if type_id == OFS_DELTA: - i = 0 + i = data_offset delta_offset = 0 s = 7 - while c & 0x80: + while True: c = ord(data[i]) - i += 1 delta_offset += (c & 0x7f) << s + i += 1 + if not (c & 0x80): + break s += 7 # END character loop + total_offset = i if as_stream: - stream = DecompressMemMapReader(buffer(data, i), False, uncomp_size) - return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) + stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) + obj = ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) else: - return ODeltaPackInfo(type_id, uncomp_size, delta_offset) + obj = ODeltaPackInfo(type_id, uncomp_size, delta_offset) # END handle stream elif type_id == REF_DELTA: - ref_sha = data[:20] + total_offset = data_offset+20 + ref_sha = data[data_offset:total_offset] + if as_stream: - stream = DecompressMemMapReader(buffer(data, 20), False, uncomp_size) - return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) + stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) + obj = ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) else: - return ODeltaPackInfo(type_id, uncomp_size, ref_sha) + obj = ODeltaPackInfo(type_id, uncomp_size, ref_sha) # END handle stream else: + total_offset = data_offset # assume its a base object if as_stream: # if no size is given, it will read the header on first access - stream = DecompressMemMapReader(buffer(data, data_offset), False) - return OPackStream(type_id, uncomp_size, stream) + stream = DecompressMemMapReader(buffer(data, data_offset), False, uncomp_size) + obj = OPackStream(type_id, uncomp_size, stream) else: - return OPackInfo(type_id, uncomp_size) + obj = OPackInfo(type_id, uncomp_size) # END handle as_stream # END handle type id + return total_offset, obj + #} END utilities @@ -267,7 +281,8 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 + _first_object_offset = 3*4 # header bytes + _footer_size = 20 # final sha def __init__(self, packpath): self._packpath = packpath @@ -287,16 +302,28 @@ def _set_cache_(self, attr): assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version # END handle header - def _iter_objects(self, start_offset, as_stream): + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data - size = len(data) + content_size = len(data) - self._footer_size cur_offset = start_offset or self._first_object_offset - while cur_offset < size: - ostream = pack_object_at(buffer(data, cur_offset), True) - # TODO: Decompressor needs to track the size of bytes actually decompressed + null = NullStream() + while cur_offset < content_size: + header_offset, ostream = pack_object_at(buffer(data, cur_offset), True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + cur_offset += header_offset + ostream.stream.compressed_bytes_read() + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream # END until we have read everything #{ Interface @@ -329,6 +356,15 @@ def stream(self, offset): :return: OPackStream instance, the actual type differs depending on the type_id attribute""" raise NotImplementedError() + def stream_iter(self, start_offset=0): + """:return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + :note: Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + #} END Read-Database like Interface diff --git a/stream.py b/stream.py index fb58125bf..898059def 100644 --- a/stream.py +++ b/stream.py @@ -17,6 +17,21 @@ #{ RO Streams +class NullStream(object): + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + + class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand @@ -144,7 +159,9 @@ def compressed_bytes_read(self): self._br = self._s # END handle stream scrubbing - return self._cbr - len(self._zip.unused_data) + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading @@ -243,7 +260,17 @@ def read(self, size=-1): if dat: dcompdat = dat + dcompdat + # END prepend our cached data + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. + # Recursively resolve that. + # Note: dcompdat can be empty even though we still appear to have bytes + # to read, if we are called by compressed_bytes_read - it manipulates + # us to empty the stream + if dcompdat and len(dcompdat) < size and self._br < self._s: + dcompdat += self.read(size-len(dcompdat)) + # END handle special case return dcompdat #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index 14d8a2496..a5234a6ce 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -52,6 +52,9 @@ def _assert_pack_file(self, pack, version, size): assert pack.size() == size assert len(pack.checksum()) == 20 + objs = list(pack.stream_iter()) + assert len(objs) == size + def test_pack_index(self): # check version 1 and 2 From ca8236451439490670822103b475df9925884e32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 21:57:22 +0200 Subject: [PATCH 0019/1790] Implemented offset based pack object collection including test, next up is the actual stream delta handling --- base.py | 59 +++++++++++------ fun.py | 9 +++ pack.py | 157 +++++++++++++++++++++++++++++++--------------- test/test_base.py | 12 +++- test/test_pack.py | 29 ++++++++- 5 files changed, 191 insertions(+), 75 deletions(-) diff --git a/base.py b/base.py index d12181229..aa917858d 100644 --- a/base.py +++ b/base.py @@ -5,7 +5,10 @@ zlib ) -from fun import type_id_to_type_map +from fun import ( + type_id_to_type_map, + type_to_type_id_map + ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', @@ -41,6 +44,10 @@ def sha(self): @property def type(self): return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] @property def size(self): @@ -50,28 +57,40 @@ def size(self): class OPackInfo(tuple): """As OInfo, but provides a type_id property to retrieve the numerical type id, and - does not include a sha""" + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the abosolute + location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - def __new__(cls, type, size): - return tuple.__new__(cls, (type, size)) + def __new__(cls, packoffset, dataoffset, type, size): + return tuple.__new__(cls, (packoffset, dataoffset, type, size)) def __init__(self, *args): tuple.__init__(self) #{ Interface + @property + def pack_offset(self): + return self[0] + + @property + def data_offset(self): + return self[1] + @property def type(self): - return type_id_to_type_map[self[0]] + return type_id_to_type_map[self[2]] @property def type_id(self): - return self[0] + return self[2] @property def size(self): - return self[1] + return self[3] #} END interface @@ -79,17 +98,17 @@ def size(self): class ODeltaPackInfo(OPackInfo): """Adds delta specific information, Either the 20 byte sha which points to some object in the database, - or the base_offset, being an offset into the pack at which our base - can be found""" + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" __slots__ = tuple() - def __new__(cls, type, size, delta_info): - return tuple.__new__(cls, (type, size, delta_info)) + def __new__(cls, packoffset, dataoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info)) #{ Interface @property def delta_info(self): - return self[2] + return self[4] #} END interface @@ -123,17 +142,17 @@ class OPackStream(OPackInfo): is provided""" __slots__ = tuple() - def __new__(cls, type, size, stream, *args): + def __new__(cls, packoffset, dataoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (type, size, stream)) + return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[2].read(size) + return self[4].read(size) @property def stream(self): - return self[2] + return self[4] #} END stream reader interface @@ -141,17 +160,17 @@ class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - def __new__(cls, type, size, delta_info, stream): - return tuple.__new__(cls, (type, size, delta_info, stream)) + def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[3].read(size) + return self[5].read(size) @property def stream(self): - return self[3] + return self[5] #} END stream reader interface diff --git a/fun.py b/fun.py index bf223e904..7999c0a92 100644 --- a/fun.py +++ b/fun.py @@ -24,6 +24,15 @@ REF_DELTA : "REF_DELTA" # REFERENCE DELTA } +type_to_type_id_map = dict( + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA + ) + # used when dealing with larger streams chunk_size = 1000*1000 diff --git a/pack.py b/pack.py index fa5e43b31..8de2a47db 100644 --- a/pack.py +++ b/pack.py @@ -1,4 +1,7 @@ """Contains PackIndexFile and PackFile implementations""" +from gitdb.exc import ( + BadObject, + ) from util import ( LockedFD, LazyMixin, @@ -31,67 +34,68 @@ __all__ = ('PackIndexFile', 'PackFile') +_delta_types = (OFS_DELTA, REF_DELTA) #{ Utilities -def pack_object_at(data, as_stream): +def pack_object_at(data, offset, as_stream): """ - :return: tuple(num_header_bytes, PackInfo|PackStream) - Tuple of number of additional bytes read from data until the data stream begins - and object of the correct type according to the type of the object. + :return: PackInfo|PackStream + an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. - :param data: random accessable data at which the header of an object can be read + :param data: random accessable data containing all required information + :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read - the data, otherwise you receive an info object only - :note: a bit redundant, but it needs to be as fast as possible !""" - type_id, uncomp_size, data_offset = pack_object_header_info(data) - total_offset = None # set later, actual offset until data stream begins - obj = None + the data, otherwise you receive an info object only""" + ldata = len(data) # debug + data = buffer(data, offset) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA if type_id == OFS_DELTA: - i = data_offset - delta_offset = 0 - s = 7 - while True: + i = data_rela_offset + c = ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: c = ord(data[i]) - delta_offset += (c & 0x7f) << s i += 1 - if not (c & 0x80): - break - s += 7 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) # END character loop - total_offset = i - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) - obj = ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) - else: - obj = ODeltaPackInfo(type_id, uncomp_size, delta_offset) - # END handle stream + delta_info = delta_offset + total_rela_offset = i + # REF DELTA elif type_id == REF_DELTA: - total_offset = data_offset+20 - ref_sha = data[data_offset:total_offset] - - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) - obj = ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) - else: - obj = ODeltaPackInfo(type_id, uncomp_size, ref_sha) - # END handle stream + total_rela_offset = data_rela_offset+20 + ref_sha = data[data_rela_offset:total_rela_offset] + delta_info = ref_sha + # BASE OBJECT else: - total_offset = data_offset # assume its a base object - if as_stream: - # if no size is given, it will read the header on first access - stream = DecompressMemMapReader(buffer(data, data_offset), False, uncomp_size) - obj = OPackStream(type_id, uncomp_size, stream) - else: - obj = OPackInfo(type_id, uncomp_size) - # END handle as_stream + total_rela_offset = data_rela_offset # END handle type id - return total_offset, obj - + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + if delta_info is None: + return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream) + else: + return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return OPackInfo(offset, abs_data_offset, type_id, uncomp_size) + else: + return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream + + #} END utilities @@ -310,12 +314,12 @@ def _iter_objects(self, start_offset, as_stream=True): null = NullStream() while cur_offset < content_size: - header_offset, ostream = pack_object_at(buffer(data, cur_offset), True) + ostream = pack_object_at(data, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += header_offset + ostream.stream.compressed_bytes_read() + cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand @@ -326,7 +330,7 @@ def _iter_objects(self, start_offset, as_stream=True): yield ostream # END until we have read everything - #{ Interface + #{ Pack Information def size(self): """:return: The amount of objects stored in this pack""" @@ -340,7 +344,58 @@ def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._data[-20:] - #} END interface + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + :param offset: specifies the first byte of the object within this pack""" + out = list() + while True: + ostream = pack_object_at(self._data, offset, True) + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out + + def to_delta_stream(self, stream_list): + """Convert the given list of streams into a stream which resolves deltas + (if availble) when reading from it. + :param stream_list: one or more stream objects. If the first stream is a Delta, + there must be at least two streams in the list. The list's last stream + must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled due to a missing base object""" + if len(stream_list) == 1: + if stream_list[0].type_id in _delta_types: + raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type) + # its an object, no need to resolve anything + return stream_list[0] + # END single object special handling + + if stream_list[-1].type_id in _delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + # just create the respective stream wrapper + raise NotImplementedError() + + + #} END pack specific #{ Read-Database like Interface @@ -348,13 +403,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - raise NotImplementedError() + return pack_object_at(self._data, offset or self._first_object_offset, False) def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - raise NotImplementedError() + return pack_object_at(self._data, offset or self._first_object_offset, True) def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing @@ -390,12 +445,14 @@ def _iter_objects(self, as_stream): def info(self, sha): """Retrieve information about the object identified by the given sha :param sha: 20 byte sha1 + :raise BadObject: :return: OInfo instance""" raise NotImplementedError() def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 + :raise BadObject: :return: OStream instance""" raise NotImplementedError() diff --git a/test/test_base.py b/test/test_base.py index f7ddebaa2..524bf3054 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -21,22 +21,28 @@ def test_streams(self): # test info sha = NULL_HEX_SHA s = 20 + blob_id = 3 + info = OInfo(sha, str_blob_type, s) assert info.sha == sha assert info.type == str_blob_type + assert info.type_id == blob_id assert info.size == s # test pack info # provides type_id - blob_id = 3 - pinfo = OPackInfo(blob_id, s) + pinfo = OPackInfo(0, 1, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + assert pinfo.data_offset == 1 - dpinfo = ODeltaPackInfo(blob_id, s, sha) + dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + assert dpinfo.data_offset == 1 # test ostream diff --git a/test/test_pack.py b/test/test_pack.py index a5234a6ce..f2b0f9481 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -52,8 +52,33 @@ def _assert_pack_file(self, pack, version, size): assert pack.size() == size assert len(pack.checksum()) == 20 - objs = list(pack.stream_iter()) - assert len(objs) == size + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.data_offset == stream.data_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = pack.to_delta_stream(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we havent resolved ( as we are without an index ) + continue + # END get deltastream + # END for each object + assert num_obj == size def test_pack_index(self): From 84c4e5a33aac3010d197a3a92adbff52a83969da Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 00:39:17 +0200 Subject: [PATCH 0020/1790] initial research on possible delta-apply algorithms. True streaming appears only possible if delta opcodes are acessing only sequential memory, but through mmaps, it should still be possible to obtain decent performance even on big files --- pack.py | 6 ++-- stream.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ test/test_pack.py | 8 +++++ 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/pack.py b/pack.py index 8de2a47db..314ae07c1 100644 --- a/pack.py +++ b/pack.py @@ -25,7 +25,8 @@ ) from stream import ( DecompressMemMapReader, - NullStream + DeltaApplyReader, + NullStream, ) from struct import ( @@ -49,7 +50,6 @@ def pack_object_at(data, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - ldata = len(data) # debug data = buffer(data, offset) type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -392,7 +392,7 @@ def to_delta_stream(self, stream_list): # END check stream # just create the respective stream wrapper - raise NotImplementedError() + return DeltaApplyReader(stream_list) #} END pack specific diff --git a/stream.py b/stream.py index 898059def..735924845 100644 --- a/stream.py +++ b/stream.py @@ -272,7 +272,82 @@ def read(self, size=-1): dcompdat += self.read(size-len(dcompdat)) # END handle special case return dcompdat + + +class DeltaApplyReader(LazyMixin): + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_streams", # tuple of our stream objects + "_readers", # list of read methods from our streams + "_mm_target", # memory map of the delta-applied data + ) + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._streams = tuple(stream_list) + self._readers = None # TODO + + def _set_cache_(self, attr): + """If we are here, we apply the actual deltas""" + # fill in delta info structures, providing the source and target buffer + # sizes. + # Allocate private memory map big enough to hold the first base buffer + # It can be swapped out if it is too large. We need random access to it + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + + # NOTE: on py pre 2.5, all memory maps must actually be some kind + # of memory buffer,like StringIO ( ouch ;) ) + + + + # TODO: Once that works, figure out the ordering of the opcodes. If they + # are always in-order/sequential, an alternate implementation could + # use stream access only. Of course this would mean we would read + # all deltas in advance, analyse the opcode ranges to determine a final + # concatenated opcode list which indicates what to copy from which delta + # to which position. This preprocessing would allow true streaming + + def read(self, size=0): + # pass the call to our lazy-loaded delta-applied data + return self._mm_target.read(size) + #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index f2b0f9481..d972bef14 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -77,6 +77,14 @@ def _assert_pack_file(self, pack, version, size): # which we havent resolved ( as we are without an index ) continue # END get deltastream + + # TODO: TestStream._assert_stream_reader does that already, should + # be used instead + # read all + dstream.read() + + # read chunks + # END for each object assert num_obj == size From 6a4eee20486eca91d06d7ba1420c8a31bcf0f4a8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 11:20:06 +0200 Subject: [PATCH 0021/1790] initial version of delta-apply, but more pedandic testing is required --- fun.py | 89 ++++++++++++++++++++++++++++++++++++++++- stream.py | 100 +++++++++++++++++++++++++++++++++++++++------- test/test_pack.py | 6 +-- util.py | 13 ++++++ 4 files changed, 188 insertions(+), 20 deletions(-) diff --git a/fun.py b/fun.py index 7999c0a92..52688ba7b 100644 --- a/fun.py +++ b/fun.py @@ -9,6 +9,7 @@ from util import zlib decompressobj = zlib.decompressobj +import mmap # INVARIANTS OFS_DELTA = 6 @@ -34,7 +35,7 @@ ) # used when dealing with larger streams -chunk_size = 1000*1000 +chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', 'write_object' ) @@ -83,6 +84,26 @@ def pack_object_header_info(data): raise BadObjectType(type_id) # END handle exceptions +def msb_size(data, offset=0): + """:return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i+offset, size + def write_object(type, size, read, write, chunk_size=chunk_size): """Write the object as identified by type, size and source_stream into the target_stream @@ -111,8 +132,15 @@ def stream_copy(read, write, size, chunk_size): # WRITE ALL DATA UP TO SIZE while True: cs = min(chunk_size, size-dbw) - data_len = write(read(cs)) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) dbw += data_len + write(data) if data_len < cs or dbw == size: break # END check for stream end @@ -120,5 +148,62 @@ def stream_copy(read, write, size, chunk_size): return dbw +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): + """Apply data from a delta buffer using a source buffer to the target file, + which will be written to + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf: random access delta data + :param target_file: file like object to write the result to + :note: transcribed to python from the similar routine in patch-delta.c""" + i = 0 + twrite = target_file.write + db = delta_buf + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += i + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + # maybe skip this check ? + if (cp_off + cp_size < cp_size or + cp_off + cp_size > src_buf_size): + break + twrite(src_buf[cp_off:cp_off+cp_size]) + elif c: + twrite(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + #} END routines diff --git a/stream.py b/stream.py index 735924845..5ced7ada0 100644 --- a/stream.py +++ b/stream.py @@ -4,7 +4,14 @@ import mmap import os +from fun import ( + msb_size, + stream_copy, + apply_delta_data + ) + from util import ( + allocate_memory, LazyMixin, make_sha, write, @@ -300,9 +307,11 @@ class DeltaApplyReader(LazyMixin): * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( - "_streams", # tuple of our stream objects - "_readers", # list of read methods from our streams + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read ) def __init__(self, stream_list): @@ -311,31 +320,81 @@ def __init__(self, stream_list): base object onto which to apply the deltas""" assert len(stream_list) > 1, "Need at least one delta and one base stream" - self._streams = tuple(stream_list) - self._readers = None # TODO + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" # fill in delta info structures, providing the source and target buffer # sizes. + buffer_offset_list = list() + final_target_size = None + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + if final_target_size is None: + final_target_size = target_size + # END set final target size + buffer_offset_list.append((buffer(buf, offset), offset)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + # Allocate private memory map big enough to hold the first base buffer - # It can be swapped out if it is too large. We need random access to it + # We need random access to it + bbuf = allocate_memory(base_size) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final # target buffer is smaller than our allocated space, we just use parts - # of it + # of it upon return. + tbuf = allocate_memory(target_size) # for each delta to apply, memory map the decompressed delta and # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. - - # NOTE: on py pre 2.5, all memory maps must actually be some kind - # of memory buffer,like StringIO ( ouch ;) ) - - + for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + + ################################################################ + apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf) + ################################################################ + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size # TODO: Once that works, figure out the ordering of the opcodes. If they # are always in-order/sequential, an alternate implementation could @@ -344,10 +403,21 @@ def _set_cache_(self, attr): # concatenated opcode list which indicates what to copy from which delta # to which position. This preprocessing would allow true streaming - def read(self, size=0): - # pass the call to our lazy-loaded delta-applied data - return self._mm_target.read(size) - + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=os.SEEK_SET): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + self._size #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index d972bef14..5786fbf72 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -78,12 +78,12 @@ def _assert_pack_file(self, pack, version, size): continue # END get deltastream - # TODO: TestStream._assert_stream_reader does that already, should - # be used instead # read all - dstream.read() + assert len(dstream.read()) # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map # END for each object assert num_obj == size diff --git a/util.py b/util.py index 5c2bb540b..f10b71b12 100644 --- a/util.py +++ b/util.py @@ -94,6 +94,19 @@ def stream_copy(source, destination, chunk_size=512*1024): # END reading output stream return br +def allocate_memory(size): + """:return: a file-protocol accessible memory block of the given size""" + try: + return mmap.mmap(-1, size) # read-write by default + except EnvironmentError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return cStringIO.StringIO("\0"*size) + # END handle memory allocation + + def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd :param fd: file descriptor opened for reading From f4b6e272963fefdb1e372b87a09e7c74680d6b52 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 13:24:41 +0200 Subject: [PATCH 0022/1790] Implemented main PackEntity object retrieval method and moved constructor for delta_streams out of the PackFile, into the stream itself where it belongs. All this is still to be tested --- fun.py | 2 + pack.py | 145 ++++++++++++++++++++++++++++++++++------------ stream.py | 49 +++++++++++++++- test/test_pack.py | 9 ++- 4 files changed, 165 insertions(+), 40 deletions(-) diff --git a/fun.py b/fun.py index 52688ba7b..cbc37b2f0 100644 --- a/fun.py +++ b/fun.py @@ -14,6 +14,8 @@ # INVARIANTS OFS_DELTA = 6 REF_DELTA = 7 +delta_types = (OFS_DELTA, REF_DELTA) + type_id_to_type_map = { 0 : "", # EXT 1 1 : "commit", diff --git a/pack.py b/pack.py index 314ae07c1..811acaf98 100644 --- a/pack.py +++ b/pack.py @@ -5,19 +5,24 @@ from util import ( LockedFD, LazyMixin, - file_contents_ro, - unpack_from + unpack_from, + file_contents_ro, ) from fun import ( pack_object_header_info, + type_id_to_type_map, stream_copy, chunk_size, + delta_types, OFS_DELTA, - REF_DELTA + REF_DELTA, + msb_size ) -from base import ( +from base import ( # Amazing ! + OInfo, + OStream, OPackInfo, OPackStream, ODeltaPackInfo, @@ -35,7 +40,7 @@ __all__ = ('PackIndexFile', 'PackFile') -_delta_types = (OFS_DELTA, REF_DELTA) + #{ Utilities @@ -95,8 +100,6 @@ def pack_object_at(data, offset, as_stream): # END handle info # END handle stream - - #} END utilities @@ -355,6 +358,7 @@ def collect_streams(self, offset): the last one is either a full object, or a REF_Delta stream. The latter type needs its reference object to be locked up in an ODB to form a valid delta chain. + If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() while True: @@ -370,31 +374,7 @@ def collect_streams(self, offset): # END handle type # END while chaining streams return out - - def to_delta_stream(self, stream_list): - """Convert the given list of streams into a stream which resolves deltas - (if availble) when reading from it. - :param stream_list: one or more stream objects. If the first stream is a Delta, - there must be at least two streams in the list. The list's last stream - must be a non-delta stream. - :return: Non-Delta OPackStream object whose stream can be used to obtain - the decompressed resolved data - :raise ValueError: if the stream list cannot be handled due to a missing base object""" - if len(stream_list) == 1: - if stream_list[0].type_id in _delta_types: - raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type) - # its an object, no need to resolve anything - return stream_list[0] - # END single object special handling - - if stream_list[-1].type_id in _delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) - # END check stream - - # just create the respective stream wrapper - return DeltaApplyReader(stream_list) - - + #} END pack specific #{ Read-Database like Interface @@ -437,8 +417,58 @@ def __init__(self, basename): self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + def _iter_objects(self, as_stream): - raise NotImplementedError + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + raise NotImplementedError() + + def _object(self, sha, as_stream): + """:return: OInfo or OStream object providing information about the given sha""" + # its a little bit redundant here, but it needs to be efficient + offset = self._index.offset(self._sha_to_index(sha)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + streams[0].seek(0) # assure it can be read by the delta reader + dstream = DeltaApplyReader.new(streams) + + return OStream(sha, dstream.type, target_size, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + + return OInfo(sha, streams[-1].type, target_size) + # END handle stream #{ Read-Database like Interface @@ -447,14 +477,14 @@ def info(self, sha): :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance""" - raise NotImplementedError() + return self._object(sha, as_stream=False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance""" - raise NotImplementedError() + return self._object(sha, as_stream=True) #} END Read-Database like Interface @@ -470,4 +500,47 @@ def stream_iter(self): OStream instances""" return self._iter_objects(as_stream=True) - #} Interface + def collect_streams_at_offset(self, offset): + """As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + + + #} END interface diff --git a/stream.py b/stream.py index 5ced7ada0..10a8e057c 100644 --- a/stream.py +++ b/stream.py @@ -7,7 +7,8 @@ from fun import ( msb_size, stream_copy, - apply_delta_data + apply_delta_data, + delta_types ) from util import ( @@ -19,7 +20,7 @@ zlib ) -__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') #{ RO Streams @@ -418,6 +419,50 @@ def seek(self, offset, whence=os.SEEK_SET): raise ValueError("Can only seek to position 0") # END handle offset self._size + + #{ Interface + + @classmethod + def new(cls, stream_list): + """Convert the given list of streams into a stream which resolves deltas + when reading from it. + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + return cls(stream_list) + + #} END interface + + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index 5786fbf72..1f860961e 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,6 +5,10 @@ with_packs_rw, fixture_path ) +from gitdb.stream import ( + DeltaApplyReader + ) + from gitdb.pack import ( PackIndexFile, PackFile @@ -71,15 +75,16 @@ def _assert_pack_file(self, pack, version, size): # read the stream try: - dstream = pack.to_delta_stream(streams) + dstream = DeltaApplyReader.new(streams) except ValueError: # ignore these, old git versions use only ref deltas, # which we havent resolved ( as we are without an index ) + # Also ignore non-delta streams continue # END get deltastream # read all - assert len(dstream.read()) + assert len(dstream.read()) # read chunks # NOTE: the current implementation is safe, it basically transfers From 001f030cc8c87407d30fe87fe24929cc1edb8f48 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 16:51:39 +0200 Subject: [PATCH 0023/1790] Initial implementation of stream validation - this is the final hurdle, if that works ( which it doesn't for yet for everything ), than the pack reading would officially work --- fun.py | 7 ++- pack.py | 124 ++++++++++++++++++++++++++++++++++++++-------- stream.py | 2 +- test/test_pack.py | 38 ++++++++++++-- util.py | 15 ------ 5 files changed, 146 insertions(+), 40 deletions(-) diff --git a/fun.py b/fun.py index cbc37b2f0..1466a78bb 100644 --- a/fun.py +++ b/fun.py @@ -106,6 +106,11 @@ def msb_size(data, offset=0): raise AssertionError("Could not find terminating MSB byte in data stream") return i+offset, size +def loose_object_header(type, size): + """:return: string representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return "%s %i\0" % (type, size) + def write_object(type, size, read, write, chunk_size=chunk_size): """Write the object as identified by type, size and source_stream into the target_stream @@ -120,7 +125,7 @@ def write_object(type, size, read, write, chunk_size=chunk_size): tbw = 0 # total num bytes written # WRITE HEADER: type SP size NULL - tbw += write("%s %i\0" % (type, size)) + tbw += write(loose_object_header(type, size)) tbw += stream_copy(read, write, size, chunk_size) return tbw diff --git a/pack.py b/pack.py index 811acaf98..b1f2f1e05 100644 --- a/pack.py +++ b/pack.py @@ -3,6 +3,7 @@ BadObject, ) from util import ( + zlib, LockedFD, LazyMixin, unpack_from, @@ -12,6 +13,7 @@ from fun import ( pack_object_header_info, type_id_to_type_map, + write_object, stream_copy, chunk_size, delta_types, @@ -31,6 +33,7 @@ from stream import ( DecompressMemMapReader, DeltaApplyReader, + Sha1Writer, NullStream, ) @@ -38,7 +41,8 @@ pack, ) -__all__ = ('PackIndexFile', 'PackFile') +import os +__all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -237,6 +241,10 @@ def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._data[-40:-20] @@ -288,8 +296,8 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 # header bytes - _footer_size = 20 # final sha + first_object_offset = 3*4 # header bytes + footer_size = 20 # final sha def __init__(self, packpath): self._packpath = packpath @@ -312,8 +320,8 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data - content_size = len(data) - self._footer_size - cur_offset = start_offset or self._first_object_offset + content_size = len(data) - self.footer_size + cur_offset = start_offset or self.first_object_offset null = NullStream() while cur_offset < content_size: @@ -343,10 +351,18 @@ def version(self): """:return: the version of this pack""" return self._version + def data(self): + """:return: read-only data of this pack. It provides random access and usually + is a memory map""" + return self._data + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._data[-20:] - + + def path(self): + """:return: path to the packfile""" + return self._packpath #} END pack information #{ Pack Specific @@ -383,13 +399,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self._first_object_offset, False) + return pack_object_at(self._data, offset or self.first_object_offset, False) def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self._first_object_offset, True) + return pack_object_at(self._data, offset or self.first_object_offset, True) def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing @@ -403,7 +419,7 @@ def stream_iter(self, start_offset=0): #} END Read-Database like Interface -class PackFileEntity(object): +class PackEntity(object): """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" @@ -412,11 +428,12 @@ class PackFileEntity(object): IndexFileCls = PackIndexFile PackFileCls = PackFile - def __init__(self, basename): + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) @@ -426,12 +443,20 @@ def _sha_to_index(self, sha): def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" - raise NotImplementedError() - - def _object(self, sha, as_stream): - """:return: OInfo or OStream object providing information about the given sha""" + indexfile = self._index + _object = self._object + for index in xrange(indexfile.size()): + sha = indexfile.sha(index) + yield _object(sha, as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" # its a little bit redundant here, but it needs to be efficient - offset = self._index.offset(self._sha_to_index(sha)) + if index < 0: + index = self._sha_to_index(sha) + offset = self._index.offset(index) type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) if as_stream: if type_id not in delta_types: @@ -447,7 +472,7 @@ def _object(self, sha, as_stream): offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - streams[0].seek(0) # assure it can be read by the delta reader + streams[0].stream.seek(0) # assure it can be read by the delta reader dstream = DeltaApplyReader.new(streams) return OStream(sha, dstream.type, target_size, dstream) @@ -476,20 +501,79 @@ def info(self, sha): """Retrieve information about the object identified by the given sha :param sha: 20 byte sha1 :raise BadObject: - :return: OInfo instance""" + :return: OInfo instance, with 20 byte sha""" return self._object(sha, as_stream=False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: - :return: OStream instance""" + :return: OStream instance, with 20 byte sha""" return self._object(sha, as_stream=True) #} END Read-Database like Interface #{ Interface - + + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """Verify that the stream at the given sha is valid. + :param sha: 20 byte sha1 of the object whose stream to verify + :param use_crc: if True, the index' crc for the sha is used to determine + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + index = self._sha_to_index(sha) + offset = self._index.offset(index) + pack_data = self._pack.data() + next_index = min(self._index.size()-1, index+1) + next_offset = 0 + if next_index == index: + next_offset = len(pack_data) - self._pack.footer_size + else: + next_offset = self._index.offset(next_index) + # END get next offset + crc_value = self._index.crc(index) + + this_crc_value = 0 + crc_update = zlib.crc32 + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + cur_pos = offset + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + cur_pos += size + # END window size loop + + assert this_crc_value == crc_value + return this_crc_value == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + return True + def info_iter(self): """:return: Iterator over all objects in this pack. The iterator yields OInfo instances""" diff --git a/stream.py b/stream.py index 10a8e057c..903aeb109 100644 --- a/stream.py +++ b/stream.py @@ -474,7 +474,7 @@ class Sha1Writer(object): __slots__ = "sha1" def __init__(self): - self.sha1 = make_sha("") + self.sha1 = make_sha() #{ Stream Interface diff --git a/test/test_pack.py b/test/test_pack.py index 1f860961e..03b162178 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -10,10 +10,16 @@ ) from gitdb.pack import ( + PackEntity, PackIndexFile, PackFile ) + +from gitdb.fun import ( + delta_types, + ) from gitdb.util import to_bin_sha +from itertools import izip import os @@ -84,7 +90,7 @@ def _assert_pack_file(self, pack, version, size): # END get deltastream # read all - assert len(dstream.read()) + assert len(dstream.read()) == dstream.size # read chunks # NOTE: the current implementation is safe, it basically transfers @@ -109,8 +115,34 @@ def test_pack(self): # END for each pack to test def test_pack_entity(self): - # TODO: - pass + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + print packfile + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + + count = 0 + for info, stream in izip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.sha == stream.sha + assert len(info.sha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # verify the stream + print info + assert entity.is_valid_stream(info.sha, use_crc=True) + #assert entity.is_valid_stream(info.sha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets diff --git a/util.py b/util.py index f10b71b12..c460969ca 100644 --- a/util.py +++ b/util.py @@ -79,21 +79,6 @@ def make_sha(source=''): sha1 = sha.sha(source) return sha1 -def stream_copy(source, destination, chunk_size=512*1024): - """Copy all data from the source stream into the destination stream in chunks - of size chunk_size - - :return: amount of bytes written""" - br = 0 - while True: - chunk = source.read(chunk_size) - destination.write(chunk) - br += len(chunk) - if len(chunk) < chunk_size: - break - # END reading output stream - return br - def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" try: From ecb18782c423bfdc45a474af2b7fbb61f62fa750 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 17:56:20 +0200 Subject: [PATCH 0024/1790] CRC verification already works for all packs, sha1 still needs some work, probably with deltified objects, there it shows whether we did it aaaaaall correctly ;) --- pack.py | 75 ++++++++++++++++++++++++++++++++++++----------- test/test_pack.py | 14 +++++---- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/pack.py b/pack.py index b1f2f1e05..4003e2742 100644 --- a/pack.py +++ b/pack.py @@ -1,6 +1,7 @@ """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, + BadObject, + UnsupportedOperation ) from util import ( zlib, @@ -41,6 +42,8 @@ pack, ) +from itertools import izip +import array import os __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -253,6 +256,21 @@ def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._data[-20:] + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + :note: return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) + + # networkbyteorder to something array likes more + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in xrange(self.size())) + # END handle version + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -419,11 +437,14 @@ def stream_iter(self, start_offset=0): #} END Read-Database like Interface -class PackEntity(object): +class PackEntity(LazyMixin): """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - __slots__ = ('_index', '_pack') + __slots__ = ( '_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) IndexFileCls = PackIndexFile PackFileCls = PackFile @@ -433,6 +454,28 @@ def __init__(self, pack_or_index_path): basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def _set_cache_(self, attr): + # currently this can only be _offset_map + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = { offsets_sorted[0] : last_offset } + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + iter_offsets_plus_one.next() + consecutive = izip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" @@ -537,33 +580,31 @@ def is_valid_stream(self, sha, use_crc=False): :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + index = self._sha_to_index(sha) offset = self._index.offset(index) - pack_data = self._pack.data() - next_index = min(self._index.size()-1, index+1) - next_offset = 0 - if next_index == index: - next_offset = len(pack_data) - self._pack.footer_size - else: - next_offset = self._index.offset(next_index) - # END get next offset + next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - this_crc_value = 0 - crc_update = zlib.crc32 - # create the current crc value, on the compressed object data # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() cur_pos = offset + this_crc_value = 0 while cur_pos < next_offset: rbound = min(cur_pos + chunk_size, next_offset) size = rbound - cur_pos - crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - assert this_crc_value == crc_value - return this_crc_value == crc_value + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value else: shawriter = Sha1Writer() stream = self._object(sha, as_stream=True) diff --git a/test/test_pack.py b/test/test_pack.py index 03b162178..d9823efa0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -15,9 +15,8 @@ PackFile ) -from gitdb.fun import ( - delta_types, - ) +from gitdb.fun import delta_types +from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha from itertools import izip import os @@ -42,6 +41,7 @@ def _assert_index_file(self, index, version, size): assert len(index.indexfile_checksum()) == 20 assert index.version() == version assert index.size() == size + assert len(index.offsets()) == size # get all data of all objects for oidx in xrange(index.size()): @@ -137,8 +137,12 @@ def test_pack_entity(self): # verify the stream print info - assert entity.is_valid_stream(info.sha, use_crc=True) - #assert entity.is_valid_stream(info.sha, use_crc=False) + try: + assert entity.is_valid_stream(info.sha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.sha, use_crc=False) # END for each info, stream tuple assert count == size From 5af5cd919aea64aeb8be8a5dc38cc6a169878399 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 18:36:33 +0200 Subject: [PATCH 0025/1790] Sha1 verification works as well, forgot to fill in the base buffer for delta-application, and fixed the broken DeltaApplyReader's seek method --- fun.py | 2 +- pack.py | 1 + stream.py | 6 +++++- test/test_pack.py | 10 +++++++--- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fun.py b/fun.py index 1466a78bb..dd9a53b5d 100644 --- a/fun.py +++ b/fun.py @@ -200,7 +200,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if (cp_off + cp_size < cp_size or cp_off + cp_size > src_buf_size): break - twrite(src_buf[cp_off:cp_off+cp_size]) + twrite(buffer(src_buf, cp_off, cp_size)) elif c: twrite(db[i:i+c]) i += c diff --git a/pack.py b/pack.py index 4003e2742..ca9dac959 100644 --- a/pack.py +++ b/pack.py @@ -611,6 +611,7 @@ def is_valid_stream(self, sha, use_crc=False): # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification return True diff --git a/stream.py b/stream.py index 903aeb109..b0fa4acb5 100644 --- a/stream.py +++ b/stream.py @@ -358,6 +358,7 @@ def _set_cache_(self, attr): # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final @@ -408,6 +409,8 @@ def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying data = self._mm_target.read(count) self._br += len(data) return data @@ -418,7 +421,8 @@ def seek(self, offset, whence=os.SEEK_SET): if offset != 0 or whence != os.SEEK_SET: raise ValueError("Can only seek to position 0") # END handle offset - self._size + self._br = 0 + self._mm_target.seek(0) #{ Interface diff --git a/test/test_pack.py b/test/test_pack.py index d9823efa0..fbcf84e39 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -90,7 +90,13 @@ def _assert_pack_file(self, pack, version, size): # END get deltastream # read all - assert len(dstream.read()) == dstream.size + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + # read chunks # NOTE: the current implementation is safe, it basically transfers @@ -119,7 +125,6 @@ def test_pack_entity(self): (self.packfile_v2_2, self.packindexfile_v2)): packfile, version, size = packinfo indexfile, version, size = indexinfo - print packfile entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile @@ -136,7 +141,6 @@ def test_pack_entity(self): assert not info.type_id in delta_types # verify the stream - print info try: assert entity.is_valid_stream(info.sha, use_crc=True) except UnsupportedOperation: From 325742cfe258436302fe0bb92462dc18a22261c7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 22:36:44 +0200 Subject: [PATCH 0026/1790] Implemented basic info and stream retrieval as well as pack file handling of PackedDB - its now operational. Next up is a performance test --- db/pack.py | 123 +++++++++++++++++++++++++++++++++++++++++-- pack.py | 30 +++++++---- test/db/test_pack.py | 45 ++++++++++++++-- test/test_pack.py | 14 +++-- 4 files changed, 193 insertions(+), 19 deletions(-) diff --git a/db/pack.py b/db/pack.py index a850e0fb2..f5ee04d6e 100644 --- a/db/pack.py +++ b/db/pack.py @@ -4,29 +4,95 @@ ObjectDBR ) +from gitdb.util import ( + to_bin_sha, + LazyMixin + ) + from gitdb.exc import ( + BadObject, UnsupportedOperation, ) +from gitdb.pack import PackEntity + +import os +import glob __all__ = ('PackedDB', ) -class PackedDB(FileDBBase, ObjectDBR): + +#{ Utilities + + +class PackedDB(FileDBBase, ObjectDBR, LazyMixin): """A database operating on a set of object packs""" + # sort the priority list every N queries + _sort_interval = 15 + def __init__(self, root_path): super(PackedDB, self).__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + # currently it can only be our _entities attribute + self._entities = list() + self.update_pack_entity_cache() + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + :note: This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + sha = to_bin_sha(sha) + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) #{ Object DB Read def has_object(self, sha): - raise NotImplementedError() + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling def info(self, sha): - raise NotImplementedError() + entity, index = self._pack_info(sha) + return entity.info_at_index(index) def stream(self, sha): - raise NotImplementedError() + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) #} END object db read @@ -39,6 +105,55 @@ def store(self, istream): raise UnsupportedOperation() def store_async(self, reader): + # TODO: add ObjectDBRW before implementing this raise NotImplementedError() #} END object db write + + + #{ Interface + + def update_pack_entity_cache(self, force=False): + """Update our cache with the acutally existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = set(item[1].pack().path() for item in self._entities) + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + #} END interface diff --git a/pack.py b/pack.py index ca9dac959..4e70c7ce9 100644 --- a/pack.py +++ b/pack.py @@ -40,6 +40,7 @@ from struct import ( pack, + unpack, ) from itertools import izip @@ -196,7 +197,7 @@ def _offset_v2(self, i): # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: - offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0] + offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset return offset @@ -291,7 +292,7 @@ def sha_to_index(self, sha): elif not c: return mid else: - lo = mid + lo = mid + 1 # END handle midpoint # END bisect return None @@ -326,13 +327,15 @@ def _set_cache_(self, attr): fd = ldb.open() self._data = file_contents_ro(fd) ldb.rollback() - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - else: + # read the header information type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) - assert type_id == "PACK", "Pack file format is invalid: %r" % type_id - assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + else: # must be '_size' or '_version' + # read header info - we do that just with a file stream + type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12)) # END handle header def _iter_objects(self, start_offset, as_stream=True): @@ -545,14 +548,23 @@ def info(self, sha): :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" - return self._object(sha, as_stream=False) + return self._object(sha, False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" - return self._object(sha, as_stream=True) + return self._object(sha, True) + + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) #} END Read-Database like Interface diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 6faff4695..d37c886eb 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -1,12 +1,51 @@ from lib import * from gitdb.db import PackedDB - +from gitdb.test.lib import fixture_path + +import os +import random + class TestPackDB(TestDBBase): @with_rw_directory @with_packs_rw def test_writing(self, path): - ldb = PackedDB(path) - # TODO + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = 2 + assert len(pdb._entities) == num_packs + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb._entities[0][1].pack().path() + new_pack_path = pack_path + "renamed" + os.rename(pack_path, new_pack_path) + pdb.update_pack_entity_cache(force=True) + assert len(pdb._entities) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_pack_entity_cache(force=True) + assert len(pdb._entities) == num_packs + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list() + for entity in (item[1] for item in pdb._entities): + for index in xrange(entity.index().size()): + + sha_list.append(entity.index().sha(index)) + # END for each index + # END for each entity + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + info = pdb.info(sha) + stream = pdb.stream(sha) + # END for each sha to query diff --git a/test/test_pack.py b/test/test_pack.py index fbcf84e39..6cfd784d4 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,9 +5,7 @@ with_packs_rw, fixture_path ) -from gitdb.stream import ( - DeltaApplyReader - ) +from gitdb.stream import DeltaApplyReader from gitdb.pack import ( PackEntity, @@ -15,6 +13,11 @@ PackFile ) +from gitdb.base import ( + OInfo, + OStream, + ) + from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha @@ -140,6 +143,11 @@ def test_pack_entity(self): # we return fully resolved items, which is implied by the sha centric access assert not info.type_id in delta_types + # try all calls + assert len(entity.collect_streams(info.sha)) + assert isinstance(entity.info(info.sha), OInfo) + assert isinstance(entity.stream(info.sha), OStream) + # verify the stream try: assert entity.is_valid_stream(info.sha, use_crc=True) From 8ab9b4f307a0672fb7b7ae29812b4ffd3d9dc8bc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 23:36:53 +0200 Subject: [PATCH 0027/1790] PackedDB: added sha_iter and size methods, these should move to the ObjectDBR actually Added performance test, packed stream reading still runs into errors, which is interesting as it dealt with the sample packs very well before --- db/pack.py | 19 +++++++++- test/db/test_pack.py | 9 ++--- test/performance/lib.py | 1 + test/performance/test_db.py | 70 +++++++++++++++++++++++++++++++++---- 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/db/pack.py b/db/pack.py index f5ee04d6e..97890ee20 100644 --- a/db/pack.py +++ b/db/pack.py @@ -28,7 +28,9 @@ class PackedDB(FileDBBase, ObjectDBR, LazyMixin): """A database operating on a set of object packs""" # sort the priority list every N queries - _sort_interval = 15 + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 def __init__(self, root_path): super(PackedDB, self).__init__(root_path) @@ -156,4 +158,19 @@ def update_pack_entity_cache(self, force=False): self._sort_entities() return True + def sha_iter(self): + """Return iterator yielding 20 byte shas for the packed objects in this data base""" + sha_list = list() + for entity in (item[1] for item in self._entities): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + """:return: amount of packed objects in this database""" + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes) #} END interface diff --git a/test/db/test_pack.py b/test/db/test_pack.py index d37c886eb..89f73f036 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -34,13 +34,8 @@ def test_writing(self, path): # bang on the cache # access the Entities directly, as there is no iteration interface # yet ( or required for now ) - sha_list = list() - for entity in (item[1] for item in pdb._entities): - for index in xrange(entity.index().size()): - - sha_list.append(entity.index().sha(index)) - # END for each index - # END for each entity + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() # hit all packs in random order random.shuffle(sha_list) diff --git a/test/performance/lib.py b/test/performance/lib.py index 03788c081..45e0ca53f 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -44,6 +44,7 @@ def setUpAll(cls): except AttributeError: pass cls.gitrepopath = resolve_or_fail(k_env_git_repo) + assert cls.gitrepopath.endswith('.git') #} END base classes diff --git a/test/performance/test_db.py b/test/performance/test_db.py index cd231b650..f6d855e20 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -1,15 +1,71 @@ """Performance tests for object store""" +from lib import ( + TestBigRepoR + ) + +from gitdb.db.pack import PackedDB import sys +import os from time import time - -from lib import ( - TestBigRepoR - ) +import random class TestGitDBPerformance(TestBigRepoR): - def test_random_access(self): - pass - # TODO: use the actual db for this + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + assert len(pdb._entities) > 1 + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + access_times = list() + for rand in range(2): + if rand: + random.shuffle(sha_list) + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + access_times.append(elapsed) + + # discard cache + del(pdb._entities) + pdb._entities + print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) + # END for each random mode + elapsed_order, elapsed_rand = access_times + + # well, its never really sequencial regarding the memory patterns, but it + # shows how well the prioriy cache performs + print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100)) + + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + # END for each function + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + stream.read() + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed) From e9c5cf3df54d0879662194f22d43a984e89b2cdf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 00:49:34 +0200 Subject: [PATCH 0028/1790] delta-apply now works after fixing a stupid type, instead of i + 1 I wrote i + i ... argh \! --- fun.py | 18 +++++++++++++----- stream.py | 24 +++++++++++------------- test/performance/test_db.py | 5 ++++- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/fun.py b/fun.py index dd9a53b5d..82b9373b2 100644 --- a/fun.py +++ b/fun.py @@ -155,7 +155,8 @@ def stream_copy(read, write, size, chunk_size): return dbw -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, + target_size): """Apply data from a delta buffer using a source buffer to the target file, which will be written to :param src_buf: random access data from which the delta was created @@ -163,6 +164,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param target_file: file like object to write the result to + :param target_size: size of the target buffer :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 twrite = target_file.write @@ -180,7 +182,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi i += 1 if (c & 0x04): cp_off |= (ord(db[i]) << 16) - i += i + i += 1 if (c & 0x08): cp_off |= (ord(db[i]) << 24) i += 1 @@ -196,14 +198,20 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if not cp_size: cp_size = 0x10000 - # maybe skip this check ? - if (cp_off + cp_size < cp_size or - cp_off + cp_size > src_buf_size): + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size or + cp_size > target_size): break twrite(buffer(src_buf, cp_off, cp_size)) + target_size -= cp_size elif c: + if c > target_size: + break twrite(db[i:i+c]) i += c + target_size -= c else: raise ValueError("unexpected delta opcode 0") # END handle command byte diff --git a/stream.py b/stream.py index b0fa4acb5..8c171b23a 100644 --- a/stream.py +++ b/stream.py @@ -327,19 +327,15 @@ def __init__(self, stream_list): def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" - # fill in delta info structures, providing the source and target buffer - # sizes. - buffer_offset_list = list() - final_target_size = None + + # prefetch information + buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: buf = dstream.read(512) # read the header information + X offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - if final_target_size is None: - final_target_size = target_size - # END set final target size - buffer_offset_list.append((buffer(buf, offset), offset)) + buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream @@ -358,7 +354,7 @@ def _set_cache_(self, attr): # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final @@ -370,7 +366,8 @@ def _set_cache_(self, attr): # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. - for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)): + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) @@ -381,15 +378,16 @@ def _set_cache_(self, attr): # read the rest from the stream. The size we give is larger than necessary stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - ################################################################ - apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf) - ################################################################ + ####################################################################### + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size) + ####################################################################### # finally, swap out source and target buffers. The target is now the # base for the next delta to apply bbuf, tbuf = tbuf, bbuf bbuf.seek(0) tbuf.seek(0) + final_target_size = target_size # END for each delta to apply # its already seeked to 0, constrain it to the actual size diff --git a/test/performance/test_db.py b/test/performance/test_db.py index f6d855e20..3948003cc 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -63,9 +63,12 @@ def test_pack_random_access(self): # retrieve stream and read all max_items = 5000 pdb_stream = pdb.stream + total_size = 0 st = time() for sha in sha_list[:max_items]: stream = pdb_stream(sha) stream.read() + total_size += stream.size elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed) + total_kib = total_size / 1000 + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) From fc6253d8428631029a9cb42830c9829692e92997 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 01:04:45 +0200 Subject: [PATCH 0029/1790] removed some extra checks in apply-delta which are indeed not required --- fun.py | 11 ++--------- stream.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/fun.py b/fun.py index 82b9373b2..d7e43717c 100644 --- a/fun.py +++ b/fun.py @@ -155,8 +155,7 @@ def stream_copy(read, write, size, chunk_size): return dbw -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, - target_size): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): """Apply data from a delta buffer using a source buffer to the target file, which will be written to :param src_buf: random access data from which the delta was created @@ -164,7 +163,6 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param target_file: file like object to write the result to - :param target_size: size of the target buffer :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 twrite = target_file.write @@ -201,17 +199,12 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size or - cp_size > target_size): + rbound > src_buf_size): break twrite(buffer(src_buf, cp_off, cp_size)) - target_size -= cp_size elif c: - if c > target_size: - break twrite(db[i:i+c]) i += c - target_size -= c else: raise ValueError("unexpected delta opcode 0") # END handle command byte diff --git a/stream.py b/stream.py index 8c171b23a..6c388a96c 100644 --- a/stream.py +++ b/stream.py @@ -379,7 +379,7 @@ def _set_cache_(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size) + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf) ####################################################################### # finally, swap out source and target buffers. The target is now the From ae6d08e1b63bf05ddcb3ee5fe027c5d829380afc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 10:10:28 +0200 Subject: [PATCH 0030/1790] Removed data-offset field from PackInfo as it is not needed in most cases. Instead, pack_at_offset returns the data-offset, slightly improving performance, and reducing memory demands --- base.py | 36 ++++++++++++++++-------------------- pack.py | 23 ++++++++++++----------- test/performance/test_db.py | 2 +- test/test_base.py | 2 -- test/test_pack.py | 1 - 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/base.py b/base.py index aa917858d..25968f371 100644 --- a/base.py +++ b/base.py @@ -64,8 +64,8 @@ class OPackInfo(tuple): location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size): - return tuple.__new__(cls, (packoffset, dataoffset, type, size)) + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset,type, size)) def __init__(self, *args): tuple.__init__(self) @@ -76,21 +76,17 @@ def __init__(self, *args): def pack_offset(self): return self[0] - @property - def data_offset(self): - return self[1] - @property def type(self): - return type_id_to_type_map[self[2]] + return type_id_to_type_map[self[1]] @property def type_id(self): - return self[2] + return self[1] @property def size(self): - return self[3] + return self[2] #} END interface @@ -102,13 +98,13 @@ class ODeltaPackInfo(OPackInfo): the pack offset of the base object""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, delta_info): - return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info)) + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) #{ Interface @property def delta_info(self): - return self[4] + return self[3] #} END interface @@ -142,17 +138,17 @@ class OPackStream(OPackInfo): is provided""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, stream, *args): + def __new__(cls, packoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream)) + return tuple.__new__(cls, (packoffset, type, size, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[4].read(size) + return self[3].read(size) @property def stream(self): - return self[4] + return self[3] #} END stream reader interface @@ -160,17 +156,17 @@ class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream): - return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream)) + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[5].read(size) + return self[4].read(size) @property def stream(self): - return self[5] + return self[4] #} END stream reader interface diff --git a/pack.py b/pack.py index 4e70c7ce9..1996d0389 100644 --- a/pack.py +++ b/pack.py @@ -55,7 +55,7 @@ def pack_object_at(data, offset, as_stream): """ - :return: PackInfo|PackStream + :return: Tuple(abs_data_offset, PackInfo|PackStream) an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. @@ -97,14 +97,14 @@ def pack_object_at(data, offset, as_stream): if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) if delta_info is None: - return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream) + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream) + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: - return OPackInfo(offset, abs_data_offset, type_id, uncomp_size) + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) else: - return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info) + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) # END handle info # END handle stream @@ -278,6 +278,7 @@ def sha_to_index(self, sha): if the sha was not found in this pack index :param sha: 20 byte sha to lookup""" first_byte = ord(sha[0]) + get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] @@ -286,7 +287,7 @@ def sha_to_index(self, sha): # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 - c = cmp(sha, self.sha(mid)) + c = cmp(sha, get_sha(mid)) if c < 0: hi = mid elif not c: @@ -346,12 +347,12 @@ def _iter_objects(self, start_offset, as_stream=True): null = NullStream() while cur_offset < content_size: - ostream = pack_object_at(data, cur_offset, True) + data_offset, ostream = pack_object_at(data, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand @@ -399,7 +400,7 @@ def collect_streams(self, offset): :param offset: specifies the first byte of the object within this pack""" out = list() while True: - ostream = pack_object_at(self._data, offset, True) + ostream = pack_object_at(self._data, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -420,13 +421,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, False) + return pack_object_at(self._data, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, True) + return pack_object_at(self._data, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing diff --git a/test/performance/test_db.py b/test/performance/test_db.py index 3948003cc..46db794fd 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -14,7 +14,6 @@ class TestGitDBPerformance(TestBigRepoR): def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - assert len(pdb._entities) > 1 # sha lookup st = time() @@ -72,3 +71,4 @@ def test_pack_random_access(self): elapsed = time() - st total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + diff --git a/test/test_base.py b/test/test_base.py index 524bf3054..c122ec4b1 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -35,14 +35,12 @@ def test_streams(self): assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - assert pinfo.data_offset == 1 dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - assert dpinfo.data_offset == 1 # test ostream diff --git a/test/test_pack.py b/test/test_pack.py index 6cfd784d4..6821097ad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -72,7 +72,6 @@ def _assert_pack_file(self, pack, version, size): stream = pack.stream(obj.pack_offset) assert info.pack_offset == stream.pack_offset - assert info.data_offset == stream.data_offset assert info.type_id == stream.type_id assert hasattr(stream, 'read') From 4503a78e699cb42935f15baa46f5947a0020911f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 12:37:42 +0200 Subject: [PATCH 0031/1790] Added initial test for putting some often-called functions into an extension module, for now its only being built by a cheap hardcoded makefile. It shows that the performance gain is rather small, bottlenecks are attr accesses, so in fact the whole type wants to be put into C to get real performance. Its not really worth it for 25% I believe --- .gitignore | 2 ++ _fun.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ makefile | 12 +++++++ pack.py | 13 ++++++++ 4 files changed, 124 insertions(+) create mode 100644 _fun.c create mode 100644 makefile diff --git a/.gitignore b/.gitignore index 0d20b6487..1a9d961a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.pyc +*.o +*.so diff --git a/_fun.c b/_fun.c new file mode 100644 index 000000000..ce9f25b16 --- /dev/null +++ b/_fun.c @@ -0,0 +1,97 @@ +#include +#include + +static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) +{ + const unsigned char *sha; + const unsigned int sha_len; + + // Note: self is only set if we are a c type. We emulate an instance method, + // hence we have to get the instance as 'first' argument + + // get instance and sha + PyObject* inst = 0; + if (!PyArg_ParseTuple(args, "Os#", &inst, &sha, &sha_len)) + return NULL; + + if (sha_len != 20) { + PyErr_SetString(PyExc_ValueError, "Sha is not 20 bytes long"); + return NULL; + } + + if( !inst){ + PyErr_SetString(PyExc_ValueError, "Cannot be called without self"); + return NULL; + } + + // read lo and hi bounds + PyObject* fanout_table = PyObject_GetAttrString(inst, "_fanout_table"); + if (!fanout_table){ + PyErr_SetString(PyExc_ValueError, "Couldn't obtain fanout table"); + return NULL; + } + + unsigned int lo = 0, hi = 0; + if (sha[0]){ + PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)(sha[0]-1)); + lo = PyInt_AS_LONG(item); + Py_DECREF(item); + } + PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)sha[0]); + hi = PyInt_AS_LONG(item); + Py_DECREF(item); + item = 0; + + Py_DECREF(fanout_table); + + // get sha query function + PyObject* get_sha = PyObject_GetAttrString(inst, "sha"); + if (!get_sha){ + PyErr_SetString(PyExc_ValueError, "Couldn't obtain sha method"); + return NULL; + } + + PyObject *sha_str = 0; + while (lo < hi) { + const int mid = (lo + hi)/2; + sha_str = PyObject_CallFunction(get_sha, "i", mid); + if (!sha_str) { + return NULL; + } + + // we really trust that string ... for speed + const int cmp = memcmp(PyString_AS_STRING(sha_str), sha, 20); + Py_DECREF(sha_str); + sha_str = 0; + + if (cmp < 0){ + lo = mid + 1; + } + else if (cmp > 0) { + hi = mid; + } + else { + Py_DECREF(get_sha); + return PyInt_FromLong(mid); + }// END handle comparison + }// END while lo < hi + + // nothing found, cleanup + Py_DECREF(get_sha); + Py_RETURN_NONE; +} + + +static PyMethodDef py_fun[] = { + { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL }, + { NULL, NULL, 0, NULL } +}; + +void init_fun(void) +{ + PyObject *m; + + m = Py_InitModule3("_fun", py_fun, NULL); + if (m == NULL) + return; +} diff --git a/makefile b/makefile new file mode 100644 index 000000000..390289625 --- /dev/null +++ b/makefile @@ -0,0 +1,12 @@ + +_fun.o: _fun.c + gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@ + +_fun.so: _fun.o + gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@ + +all: _fun.so + +clean: + -rm *.so + -rm *.o diff --git a/pack.py b/pack.py index 1996d0389..d6fe56823 100644 --- a/pack.py +++ b/pack.py @@ -23,6 +23,12 @@ msb_size ) +try: + from _fun import PackIndexFile_sha_to_index +except ImportError: + pass +# END try c module + from base import ( # Amazing ! OInfo, OStream, @@ -298,6 +304,13 @@ def sha_to_index(self, sha): # END bisect return None + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + #} END properties From 3cee78ed377b0a73febdbd772ddba2999313023e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 15:52:16 +0200 Subject: [PATCH 0032/1790] Added endurance run to check all objects in the git source repository, something like a primtive git-fsck, which indeed takes a while to run. Its useful to see the memory consumption, which must stay static in the domain of physical memory --- db/pack.py | 6 ++++- test/performance/{test_db.py => test_pack.py} | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) rename test/performance/{test_db.py => test_pack.py} (74%) diff --git a/db/pack.py b/db/pack.py index 97890ee20..2332992f2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -157,11 +157,15 @@ def update_pack_entity_cache(self, force=False): # reinitialize prioritiess self._sort_entities() return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [ item[1] for item in self._entities ] def sha_iter(self): """Return iterator yielding 20 byte shas for the packed objects in this data base""" sha_list = list() - for entity in (item[1] for item in self._entities): + for entity in self.entities(): index = entity.index() sha_by_index = index.sha for index in xrange(index.size()): diff --git a/test/performance/test_db.py b/test/performance/test_pack.py similarity index 74% rename from test/performance/test_db.py rename to test/performance/test_pack.py index 46db794fd..37abc1725 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_pack.py @@ -10,7 +10,7 @@ from time import time import random -class TestGitDBPerformance(TestBigRepoR): +class TestPackedDBPerformance(TestBigRepoR): def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -22,7 +22,6 @@ def test_pack_random_access(self): ns = len(sha_list) print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) - # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info access_times = list() @@ -39,7 +38,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) - pdb._entities + pdb.entities() print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) # END for each random mode elapsed_order, elapsed_rand = access_times @@ -72,3 +71,24 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + + print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in xrange(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + except UnsupportedOperation: + pass + # END ignore old indices + count += 1 + # END for each index + # END for each entity + elapsed = time() - st + print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + # END for each verify mode + From 6fd8d7406028d603379dc23be0ab1403785f2cd3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 19:57:28 +0200 Subject: [PATCH 0033/1790] Base implementation and stubs added for git-like db, as well as the reference db ( for the alternates implementation ) --- db/base.py | 73 +++++++++++++++++++++++++++++++++-- db/git.py | 68 +++++++++++++++++++------------- db/loose.py | 19 +++++++++ db/pack.py | 45 ++++++++++----------- db/ref.py | 61 ++++++++++++++++++++++++++++- test/db/lib.py | 1 + test/db/test_git.py | 9 +++++ test/db/test_loose.py | 5 +++ test/db/test_pack.py | 12 +++--- test/db/test_ref.py | 21 ++++++++++ test/performance/test_pack.py | 5 ++- util.py | 1 + 12 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 test/db/test_git.py create mode 100644 test/db/test_ref.py diff --git a/db/base.py b/db/base.py index 2cda0ea0a..91e82fc0b 100644 --- a/db/base.py +++ b/db/base.py @@ -9,7 +9,7 @@ ) -__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB') +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') class ObjectDBR(object): @@ -66,6 +66,14 @@ def stream_async(self, reader): # base implementation just uses the stream method repeatedly task = ChannelThreadTask(reader, str(self.stream_async), self.stream) return pool.add_task(task) + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() #} END query interface @@ -150,6 +158,63 @@ def db_path(self, rela_path): #} END interface -class CompoundDB(ObjectDBR): - """A database which delegates calls to sub-databases""" - # TODO +class CachingDB(object): + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """Call this method if the underlying data changed to trigger an update + of the internal caching structures. + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface + + +class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + + #{ ObjectDBR interface + + def has_object(self, sha): + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + raise NotImplementedError() + + def size(self): + raise NotImplementedError() + + def sha_iter(self): + raise NotImplementedError() + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) + + def update_cache(self, force=False): + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + #} END interface + + diff --git a/db/git.py b/db/git.py index 0488bc601..1953def17 100644 --- a/db/git.py +++ b/db/git.py @@ -1,33 +1,49 @@ - -from gitdb.base import ( - OInfo, - OStream - ) +from base import ( + CompoundDB, + FileDBBase, + ) from loose import LooseObjectDB +from pack import PackedDB +from ref import ReferenceDB + +from gitdb.util import LazyMixin +from gitdb.exc import InvalidDBRoot +import os -__all__ = ('GitObjectDB', ) +__all__ = ('GitDB', ) -#class GitObjectDB(CompoundDB, ObjectDBW): -class GitObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file +class GitDB(FileDBBase, CompoundDB): + """A git-style object database, which contains all objects in the 'objects' + subdirectory""" + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB - It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations - """ - def __init__(self, root_path, git): - """Initialize this instance with the root and a git command""" - super(GitObjectDB, self).__init__(root_path) - self._git = git + # Directories + packs_dir = 'packs' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super(GitDB, self).__init__(root_path) - def info(self, sha): - t = self._git.get_object_header(sha) - return OInfo(*t) + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle dbs - def stream(self, sha): - """For now, all lookup is done by git itself""" - t = self._git.stream_object_data(sha) - return OStream(*t) - diff --git a/db/loose.py b/db/loose.py index 109782fdc..b95d6f1c9 100644 --- a/db/loose.py +++ b/db/loose.py @@ -24,11 +24,13 @@ from gitdb.util import ( ENOENT, to_hex_sha, + hex_to_bin, exists, isdir, mkdir, rename, dirname, + basename, join ) @@ -186,4 +188,21 @@ def store(self, istream): istream.sha = sha return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) diff --git a/db/pack.py b/db/pack.py index 2332992f2..92fbc616b 100644 --- a/db/pack.py +++ b/db/pack.py @@ -1,7 +1,8 @@ """Module containing a database to deal with packs""" from base import ( FileDBBase, - ObjectDBR + ObjectDBR, + CachingDB ) from gitdb.util import ( @@ -18,13 +19,13 @@ import os import glob -__all__ = ('PackedDB', ) +__all__ = ('PackedDB', ) #{ Utilities -class PackedDB(FileDBBase, ObjectDBR, LazyMixin): +class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): """A database operating on a set of object packs""" # sort the priority list every N queries @@ -43,9 +44,10 @@ def __init__(self, root_path): self._st_mtime = 0 # last modification data of our root path def _set_cache_(self, attr): - # currently it can only be our _entities attribute - self._entities = list() - self.update_pack_entity_cache() + if attr == '_entities': + self._entities = list() + self.update_cache() + # END handle entities initialization def _sort_entities(self): self._entities.sort(key=lambda l: l[0], reverse=True) @@ -95,6 +97,20 @@ def info(self, sha): def stream(self, sha): entity, index = self._pack_info(sha) return entity.stream_at_index(index) + + def sha_iter(self): + sha_list = list() + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes) #} END object db read @@ -115,7 +131,7 @@ def store_async(self, reader): #{ Interface - def update_pack_entity_cache(self, force=False): + def update_cache(self, force=False): """Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones :param force: If True, the cache will be updated even though the directory @@ -162,19 +178,4 @@ def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - def sha_iter(self): - """Return iterator yielding 20 byte shas for the packed objects in this data base""" - sha_list = list() - for entity in self.entities(): - index = entity.index() - sha_by_index = index.sha - for index in xrange(index.size()): - yield sha_by_index(index) - # END for each index - # END for each entity - - def size(self): - """:return: amount of packed objects in this database""" - sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes) #} END interface diff --git a/db/ref.py b/db/ref.py index 2c63884bc..5db8d7a23 100644 --- a/db/ref.py +++ b/db/ref.py @@ -1,7 +1,64 @@ -from base import CompoundDB +from base import ( + CompoundDB, + ) +import os __all__ = ('CompoundDB', ) class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" - + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super(ReferenceDB, self).__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + # END handle dbs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + except OSError: + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = set(db.root_path() for db in self._dbs) + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + self._dbs.append(dbcls(path)) + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super(ReferenceDB, self).update_cache(force) diff --git a/test/db/lib.py b/test/db/lib.py index cf752741b..57f5eefc1 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -3,6 +3,7 @@ with_rw_directory, with_packs_rw, ZippedStoreShaWriter, + fixture_path, TestBase ) diff --git a/test/db/test_git.py b/test/db/test_git.py new file mode 100644 index 000000000..35a0d5bad --- /dev/null +++ b/test/db/test_git.py @@ -0,0 +1,9 @@ +from lib import * +from gitdb.db import GitDB + +class TestGitDB(TestBase): + + def test_reading(self): + ldb = GitDB(fixture_path('../../.git/objects') + self.fail("todo") + diff --git a/test/db/test_loose.py b/test/db/test_loose.py index 70cd7742c..536b02048 100644 --- a/test/db/test_loose.py +++ b/test/db/test_loose.py @@ -11,3 +11,8 @@ def test_writing(self, path): self._assert_object_writing(ldb) self._assert_object_writing_async(ldb) + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 89f73f036..f347f408b 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -14,22 +14,22 @@ def test_writing(self, path): # on demand, we init our pack cache num_packs = 2 - assert len(pdb._entities) == num_packs + assert len(pdb.entities()) == num_packs assert pdb._st_mtime != 0 # test pack directory changed: # packs removed - rename a file, should affect the glob - pack_path = pdb._entities[0][1].pack().path() + pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" os.rename(pack_path, new_pack_path) - pdb.update_pack_entity_cache(force=True) - assert len(pdb._entities) == num_packs - 1 + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 # packs added os.rename(new_pack_path, pack_path) - pdb.update_pack_entity_cache(force=True) - assert len(pdb._entities) == num_packs + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs # bang on the cache # access the Entities directly, as there is no iteration interface diff --git a/test/db/test_ref.py b/test/db/test_ref.py new file mode 100644 index 000000000..3b027d701 --- /dev/null +++ b/test/db/test_ref.py @@ -0,0 +1,21 @@ +from lib import * +from gitdb.db import ReferenceDB + +class TestReferenceDB(TestBase): + + @with_rw_directory + def test_writing(self, path): + # TODO: setup alternate file + alternates = + ldb = ReferenceDB(path) + + # try empty, non-existing + + # add two, one is invalid + + # remove valid + + # add valid + + self.fail("todo") + diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 37abc1725..8046c1f83 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -3,6 +3,7 @@ TestBigRepoR ) +from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB import sys @@ -39,7 +40,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) + print >> sys.stderr, "PDB: looked up %i sha in %i packs (random=%i) in %f s ( %f shas/s )" % (ns, len(pdb.entities()), rand, elapsed, ns / elapsed) # END for each random mode elapsed_order, elapsed_rand = access_times @@ -82,10 +83,10 @@ def test_pack_random_access(self): for index in xrange(entity.index().size()): try: assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 except UnsupportedOperation: pass # END ignore old indices - count += 1 # END for each index # END for each entity elapsed = time() - st diff --git a/util.py b/util.py index c460969ca..aa6db4088 100644 --- a/util.py +++ b/util.py @@ -57,6 +57,7 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir rename = os.rename dirname = os.path.dirname +basename = os.path.basename join = os.path.join read = os.read write = os.write From 92ca2e4ad606fbec7c934ad9e467a1b51fddcc92 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 20 Jun 2010 19:47:38 +0200 Subject: [PATCH 0034/1790] Implemented gitdb, it should be a fully functional git database with full read support, and the ability to write loose objects --- db/base.py | 36 ++++++++++++++++++++++++++++++------ db/git.py | 36 ++++++++++++++++++++++++++++++------ db/pack.py | 2 +- db/ref.py | 15 ++++++++++++--- test/db/lib.py | 2 +- test/db/test_git.py | 23 ++++++++++++++++++++--- test/db/test_ref.py | 42 +++++++++++++++++++++++++++++++++++++----- 7 files changed, 131 insertions(+), 25 deletions(-) diff --git a/db/base.py b/db/base.py index 91e82fc0b..35c20b7e1 100644 --- a/db/base.py +++ b/db/base.py @@ -1,13 +1,19 @@ """Contains implementations of database retrieveing objects""" from gitdb.util import ( pool, - join + join, + LazyMixin, + to_bin_sha ) +from gitdb.exc import BadObject + from async import ( ChannelThreadTask ) +from itertools import chain + __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -182,22 +188,40 @@ def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() + def _db_query(self, sha): + """:return: database containing the given 20 or 40 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it everytime a database is being queried + sha = to_bin_sha(sha) + for db in self._dbs: + if db.has_object(sha): + return db + # END for each database + raise BadObject(sha) + #{ ObjectDBR interface def has_object(self, sha): - raise NotImplementedError("To be implemented in subclass") + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions def info(self, sha): - raise NotImplementedError("To be implemented in subclass") + return self._db_query(sha).info(sha) def stream(self, sha): - raise NotImplementedError() + return self._db_query(sha).stream(sha) def size(self): - raise NotImplementedError() + """:return: total size of all contained databases""" + return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) def sha_iter(self): - raise NotImplementedError() + return chain(*(db.sha_iter() for db in self._dbs)) #} END object DBR Interface diff --git a/db/git.py b/db/git.py index 1953def17..ad9a613b3 100644 --- a/db/git.py +++ b/db/git.py @@ -1,7 +1,8 @@ from base import ( - CompoundDB, - FileDBBase, - ) + CompoundDB, + ObjectDBW, + FileDBBase + ) from loose import LooseObjectDB from pack import PackedDB @@ -13,7 +14,7 @@ __all__ = ('GitDB', ) -class GitDB(FileDBBase, CompoundDB): +class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" # Configuration @@ -22,7 +23,7 @@ class GitDB(FileDBBase, CompoundDB): ReferenceDBCls = ReferenceDB # Directories - packs_dir = 'packs' + packs_dir = 'pack' loose_dir = '' alternates_dir = os.path.join('info', 'alternates') @@ -31,19 +32,42 @@ def __init__(self, root_path): super(GitDB, self).__init__(root_path) def _set_cache_(self, attr): - if attr == '_dbs': + if attr == '_dbs' or attr == '_loose_db': self._dbs = list() + loose_db = None for subpath, dbcls in ((self.packs_dir, self.PackDBCls), (self.loose_dir, self.LooseDBCls), (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) if os.path.exists(path): self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db # END check path exists # END for each db type # should have at least one subdb if not self._dbs: raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + # END handle dbs + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + #} END objectdbw interface diff --git a/db/pack.py b/db/pack.py index 92fbc616b..af6f7ffd6 100644 --- a/db/pack.py +++ b/db/pack.py @@ -110,7 +110,7 @@ def sha_iter(self): def size(self): sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes) + return reduce(lambda x,y: x+y, sizes, 0) #} END object db read diff --git a/db/ref.py b/db/ref.py index 5db8d7a23..3a4813979 100644 --- a/db/ref.py +++ b/db/ref.py @@ -3,7 +3,7 @@ ) import os -__all__ = ('CompoundDB', ) +__all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" @@ -35,7 +35,7 @@ def _update_dbs_from_ref_file(self): ref_paths = list() try: ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] - except OSError: + except (OSError, IOError): pass # END handle alternates @@ -55,7 +55,16 @@ def _update_dbs_from_ref_file(self): # sort them to maintain order added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) for path in added_paths: - self._dbs.append(dbcls(path)) + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception, e: + # ignore invalid paths or issues + pass # END for each path to add def update_cache(self, force=False): diff --git a/test/db/lib.py b/test/db/lib.py index 57f5eefc1..2c597fbd3 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -22,7 +22,7 @@ from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_git.py b/test/db/test_git.py index 35a0d5bad..4d463f1f7 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,9 +1,26 @@ from lib import * from gitdb.db import GitDB +from gitdb.base import OStream, OInfo -class TestGitDB(TestBase): +class TestGitDB(TestDBBase): def test_reading(self): - ldb = GitDB(fixture_path('../../.git/objects') - self.fail("todo") + gdb = GitDB(fixture_path('../../.git/objects')) + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + assert gdb.size() > 200 + assert len(list(gdb.sha_iter())) == gdb.size() + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) + self._assert_object_writing_async(gdb) diff --git a/test/db/test_ref.py b/test/db/test_ref.py index 3b027d701..68d9b8116 100644 --- a/test/db/test_ref.py +++ b/test/db/test_ref.py @@ -1,21 +1,53 @@ from lib import * from gitdb.db import ReferenceDB -class TestReferenceDB(TestBase): +import os + +class TestReferenceDB(TestDBBase): + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + alt_file = open(alt_path, "wb") + for alt in alt_list: + alt_file.write(alt + "\n") + alt_file.close() @with_rw_directory def test_writing(self, path): - # TODO: setup alternate file - alternates = - ldb = ReferenceDB(path) + null_sha_bin = '\0' * 20 + null_sha_hex = "0" * 40 + + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 # try empty, non-existing + assert not rdb.has_object(null_sha_hex) + assert not rdb.has_object(null_sha_bin) + + # setup alternate file # add two, one is invalid + own_repo_path = fixture_path('../../.git/objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + assert rdb.has_object(gitdb_sha) # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 - self.fail("todo") From 92e6770be0f65393199432dcfd24f3f1b10d015e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 22 Jun 2010 10:58:45 +0200 Subject: [PATCH 0035/1790] Added MemoryDB including initial test, moved ZippedShaWriter into stream module, it was just a test helper previously --- base.py | 10 ++++++ db/__init__.py | 1 + db/mem.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ stream.py | 33 +++++++++++++++++++ test/db/lib.py | 29 +++++++++++++++- test/db/test_mem.py | 10 ++++++ test/lib.py | 27 +++------------ test/test_base.py | 4 +-- 8 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 db/mem.py create mode 100644 test/db/test_mem.py diff --git a/base.py b/base.py index 25968f371..0f4e63176 100644 --- a/base.py +++ b/base.py @@ -41,6 +41,16 @@ def __init__(self, *args): def sha(self): return self[0] + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + @property def type(self): return self[1] diff --git a/db/__init__.py b/db/__init__.py index 05d9b21b3..85a0a6874 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -1,6 +1,7 @@ from base import * from loose import * +from mem import * from pack import * from git import * from ref import * diff --git a/db/mem.py b/db/mem.py new file mode 100644 index 000000000..3bb6a339d --- /dev/null +++ b/db/mem.py @@ -0,0 +1,80 @@ +"""Contains the MemoryDatabase implementation""" +from loose import LooseObjectDB +from base import ( + ObjectDBR, + ObjectDBW + ) + +from gitdb.base import OStream +from gitdb.util import to_bin_sha +from gitdb.exc import ( + BadObject, + UnsupportedOperation + ) +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, + ) + +__all__ = ("MemoryDB", ) + +class MemoryDB(ObjectDBR, ObjectDBW): + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO + + :note: memory is currently not threadsafe, hence the async methods cannot be used + for storing""" + + def __init__(self): + super(MemoryDB, self).__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream) + + return istream + + def store_async(self, reader): + raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") + + def has_object(self, sha): + return to_bin_sha(sha) in self._cache + + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + sha = to_bin_sha(sha) + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError: + raise BadObject(sha) + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.iterkeys() diff --git a/stream.py b/stream.py index 6c388a96c..8b7e981b0 100644 --- a/stream.py +++ b/stream.py @@ -499,6 +499,39 @@ def sha(self, as_hex = False): #} END interface + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=os.SEEK_SET): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() + + class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor diff --git a/test/db/lib.py b/test/db/lib.py index 2c597fbd3..b22f9a2b8 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -20,6 +20,7 @@ from async import IteratorReader from cStringIO import StringIO +from struct import pack __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') @@ -29,9 +30,35 @@ class TestDBBase(TestBase): # data two_lines = "1234\nhello world" - all_data = (two_lines, ) + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in xrange(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), StringIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.sha) + + info = db.info(istream.sha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.sha) + assert isinstance(stream, OStream) + assert stream.sha == info.sha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW :note: requires write access to the database""" diff --git a/test/db/test_mem.py b/test/db/test_mem.py new file mode 100644 index 000000000..9e7c1190e --- /dev/null +++ b/test/db/test_mem.py @@ -0,0 +1,10 @@ +from lib import * +from gitdb.db import MemoryDB + +class TestMemoryDB(TestDBBase): + + def test_writing(self): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) diff --git a/test/lib.py b/test/lib.py index 6b25876d6..78817fea9 100644 --- a/test/lib.py +++ b/test/lib.py @@ -2,7 +2,11 @@ from gitdb import ( OStream, ) -from gitdb.stream import Sha1Writer +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter + ) + from gitdb.util import zlib import sys @@ -140,26 +144,5 @@ def _assert(self): assert self.args assert self.myarg - -class ZippedStoreShaWriter(Sha1Writer): - """Remembers everything someone writes to it""" - __slots__ = ('buf', 'zip') - def __init__(self): - Sha1Writer.__init__(self) - self.buf = StringIO() - self.zip = zlib.compressobj(1) # fastest - - def __getattr__(self, attr): - return getattr(self.buf, attr) - - def write(self, data): - alen = Sha1Writer.write(self, data) - self.buf.write(self.zip.compress(data)) - return alen - - def close(self): - self.buf.write(self.zip.flush()) - - #} END stream utilitiess diff --git a/test/test_base.py b/test/test_base.py index c122ec4b1..8d8bcc944 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -31,12 +31,12 @@ def test_streams(self): # test pack info # provides type_id - pinfo = OPackInfo(0, 1, blob_id, s) + pinfo = OPackInfo(0, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha From 9b53ab02cb44571e6167a125a5296b7c3395563f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 22 Jun 2010 11:45:26 +0200 Subject: [PATCH 0036/1790] MemoryDB: Implemented direct stream copy, allowing to flush memory db content into any other object db for permanent storage --- db/loose.py | 24 ++++++++++++++++----- db/mem.py | 33 ++++++++++++++++++++++++++++- stream.py | 51 ++++++++++++++++++++++++++++++++------------- test/db/test_mem.py | 20 ++++++++++++++++-- test/test_stream.py | 4 ++-- 5 files changed, 107 insertions(+), 25 deletions(-) diff --git a/db/loose.py b/db/loose.py index b95d6f1c9..9bcbd6aac 100644 --- a/db/loose.py +++ b/db/loose.py @@ -13,6 +13,7 @@ from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, + FDStream, Sha1Writer ) @@ -43,6 +44,7 @@ import tempfile import mmap +import sys import os @@ -153,13 +155,20 @@ def store(self, istream): if writer is None: # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - writer = FDCompressedSha1Writer(fd) + + if istream.sha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies # END handle custom writer try: try: if istream.sha is not None: - stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, @@ -175,10 +184,15 @@ def store(self, istream): writer.close() # END assure target stream is closed - sha = istream.sha or writer.sha(as_hex=True) + hexsha = None + if istream.sha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha if tmp_path: - obj_path = self.db_path(self.object_path(sha)) + obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) if not isdir(obj_dir): mkdir(obj_dir) @@ -186,7 +200,7 @@ def store(self, istream): rename(tmp_path, obj_path) # END handle dry_run - istream.sha = sha + istream.sha = hexsha return istream def sha_iter(self): diff --git a/db/mem.py b/db/mem.py index 3bb6a339d..9e3d3972d 100644 --- a/db/mem.py +++ b/db/mem.py @@ -5,7 +5,11 @@ ObjectDBW ) -from gitdb.base import OStream +from gitdb.base import ( + OStream, + IStream, + ) + from gitdb.util import to_bin_sha from gitdb.exc import ( BadObject, @@ -16,6 +20,8 @@ DecompressMemMapReader, ) +from cStringIO import StringIO + __all__ = ("MemoryDB", ) class MemoryDB(ObjectDBR, ObjectDBW): @@ -78,3 +84,28 @@ def size(self): def sha_iter(self): return self._cache.iterkeys() + + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + :note: the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existance + + ostream = self.stream(sha) + # compressed data including header + sio = StringIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/stream.py b/stream.py index 8b7e981b0..57a5a194f 100644 --- a/stream.py +++ b/stream.py @@ -25,21 +25,6 @@ #{ RO Streams -class NullStream(object): - """A stream that does nothing but providing a stream interface. - Use it like /dev/null""" - __slots__ = tuple() - - def read(self, size=0): - return '' - - def close(self): - pass - - def write(self, data): - return len(data) - - class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand @@ -113,6 +98,8 @@ def _parse_header_info(self): return type, size + #{ Interface + @classmethod def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream @@ -125,6 +112,10 @@ def new(self, m, close_on_deletion=False): type, size = inst._parse_header_info() return type, size, inst + def data(self): + """:return: random access compatible data we are working on""" + return self._m + def compressed_bytes_read(self): """:return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )""" @@ -171,6 +162,8 @@ def compressed_bytes_read(self): # from the count already return self._cbr + #} END interface + def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" @@ -567,4 +560,32 @@ def close(self): #} END stream interface +class FDStream(object): + """Simple wrapper around a file descriptor""" + __slots__ = "_fd" + def __init__(self, fd): + self._fd = fd + + def write(self, data): + return write(self._fd, data) + + def close(self): + close(self._fd) + + + +class NullStream(object): + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + #} END W streams diff --git a/test/db/test_mem.py b/test/db/test_mem.py index 9e7c1190e..4a9b7ee12 100644 --- a/test/db/test_mem.py +++ b/test/db/test_mem.py @@ -1,10 +1,26 @@ from lib import * -from gitdb.db import MemoryDB +from gitdb.db import ( + MemoryDB, + LooseObjectDB + ) class TestMemoryDB(TestDBBase): - def test_writing(self): + @with_rw_directory + def test_writing(self, path): mdb = MemoryDB() # write data self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/test/test_stream.py b/test/test_stream.py index 41f2b235a..859920719 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -50,7 +50,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle rest if isinstance(stream, DecompressMemMapReader): - assert len(stream._m) == stream.compressed_bytes_read() + assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type rewind_stream(stream) @@ -60,7 +60,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert rdata == cdata if isinstance(stream, DecompressMemMapReader): - assert len(stream._m) == stream.compressed_bytes_read() + assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type def test_decompress_reader(self): From 9e6e7d7f1f624143ef8e8fc04e55e5ca277f43ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 01:09:18 +0200 Subject: [PATCH 0037/1790] Removed redunant code due to parital reimplementation of a file descriptor stream wrapper --- db/loose.py | 14 +++++++------- stream.py | 29 ++++++++++++++++++++++++----- test/lib.py | 11 ++++++----- test/test_util.py | 10 ++++++++++ util.py | 43 +++++++++++-------------------------------- 5 files changed, 58 insertions(+), 49 deletions(-) diff --git a/db/loose.py b/db/loose.py index 9bcbd6aac..f97b98a2b 100644 --- a/db/loose.py +++ b/db/loose.py @@ -174,15 +174,15 @@ def store(self, istream): write_object(istream.type, istream.size, istream.read, writer.write, chunk_size=self.stream_chunk_size) # END handle direct stream copies - except: + finally: if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - finally: + writer.close() + # END assure target stream is closed + except: if tmp_path: - writer.close() - # END assure target stream is closed + os.remove(tmp_path) + raise + # END assure tmpfile removal on error hexsha = None if istream.sha: diff --git a/stream.py b/stream.py index 57a5a194f..3f5d0373a 100644 --- a/stream.py +++ b/stream.py @@ -560,19 +560,38 @@ def close(self): #} END stream interface + class FDStream(object): - """Simple wrapper around a file descriptor""" - __slots__ = "_fd" + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') def __init__(self, fd): self._fd = fd + self._pos = 0 def write(self, data): - return write(self._fd, data) + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos def close(self): close(self._fd) - - + class NullStream(object): """A stream that does nothing but providing a stream interface. diff --git a/test/lib.py b/test/lib.py index 78817fea9..742aa7f5c 100644 --- a/test/lib.py +++ b/test/lib.py @@ -38,11 +38,12 @@ def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) try: - return func(self, path) - except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) - raise - else: + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + raise + finally: shutil.rmtree(path) # END handle exception # END wrapper diff --git a/test/test_util.py b/test/test_util.py index 2272b53e5..6a389d27c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -88,4 +88,14 @@ def test_lockedfd(self): finally: os.remove(my_file) # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/util.py b/util.py index aa6db4088..9d1a96900 100644 --- a/util.py +++ b/util.py @@ -161,35 +161,6 @@ def _set_cache_(self, attr): in the single attribute.""" pass - -class FDStreamWrapper(object): - """A simple wrapper providing the most basic functions on a file descriptor - with the fileobject interface. Cannot use os.fdopen as the resulting stream - takes ownership""" - __slots__ = ("_fd", '_pos') - def __init__(self, fd): - self._fd = fd - self._pos = 0 - - def write(self, data): - self._pos += len(data) - os.write(self._fd, data) - - def read(self, count=0): - if count == 0: - count = os.path.getsize(self._filepath) - # END handle read everything - - bytes = os.read(self._fd, count) - self._pos += len(bytes) - return bytes - - def fileno(self): - return self._fd - - def tell(self): - return self._pos - class LockedFD(object): """This class facilitates a safe read and write operation to a file on disk. @@ -240,7 +211,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode) + fd = os.open(self._lockfilepath(), lockmode, 0600) if not write: os.close(fd) else: @@ -253,11 +224,19 @@ def open(self, write=False, stream=False): # open actual file if required if self._fd is None: # we could specify exlusive here, as we obtained the lock anyway - self._fd = os.open(self._filepath, os.O_RDONLY | binary) + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + os.remove(self._lockfilepath()) + raise + # END handle lockfile # END open descriptor for reading if stream: - return FDStreamWrapper(self._fd) + # need delayed import + from stream import FDStream + return FDStream(self._fd) else: return self._fd # END handle stream From 09275ad268e5459c8edff2cccafd0a43947cabdb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 11:45:09 +0200 Subject: [PATCH 0038/1790] Index and PackFiles do not obtain a lock anymore before reading the files in question - this won't work on read-only alternate repositories anyway, and shouldn't be necessary considering a pack is immutable --- db/loose.py | 5 +++-- pack.py | 16 ++++++---------- util.py | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/db/loose.py b/db/loose.py index f97b98a2b..c9ea3a038 100644 --- a/db/loose.py +++ b/db/loose.py @@ -23,6 +23,7 @@ ) from gitdb.util import ( + file_contents_ro_filepath, ENOENT, to_hex_sha, hex_to_bin, @@ -100,12 +101,12 @@ def _map_loose_object(self, sha): :raise BadObject: if object could not be located""" db_path = self.db_path(self.object_path(to_hex_sha(sha))) try: - fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) except OSError,e: if e.errno != ENOENT: # try again without noatime try: - fd = os.open(db_path, os.O_RDONLY) + return file_contents_ro_filepath(db_path) except OSError: raise BadObject(to_hex_sha(sha)) # didn't work because of our flag, don't try it again diff --git a/pack.py b/pack.py index d6fe56823..c66d6717e 100644 --- a/pack.py +++ b/pack.py @@ -5,10 +5,9 @@ ) from util import ( zlib, - LockedFD, LazyMixin, unpack_from, - file_contents_ro, + file_contents_ro_filepath, ) from fun import ( @@ -140,10 +139,10 @@ def _set_cache_(self, attr): elif attr == "_packfile_checksum": self._packfile_checksum = self._data[-20:] elif attr == "_data": - lfd = LockedFD(self._indexpath) - fd = lfd.open() - self._data = file_contents_ro(fd) - lfd.rollback() + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._data = file_contents_ro_filepath(self._indexpath) else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -337,10 +336,7 @@ def __init__(self, packpath): def _set_cache_(self, attr): if attr == '_data': - ldb = LockedFD(self._packpath) - fd = ldb.open() - self._data = file_contents_ro(fd) - ldb.rollback() + self._data = file_contents_ro_filepath(self._packpath) # read the header information type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) diff --git a/util.py b/util.py index 9d1a96900..f0bf3e60f 100644 --- a/util.py +++ b/util.py @@ -114,7 +114,24 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): if stream: return cStringIO.StringIO(contents) return contents - + +def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): + """Get the file contents at filepath as fast as possible + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + :note: for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY|flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: From d3a0037dd5a11459985e7dc4b6819f6292f20c13 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 15:28:22 +0200 Subject: [PATCH 0039/1790] Fixed critical issues with incorrect permissions set on files the db has written - it was only rw-- for the user that wrote them, but should be readable by everyone by default --- db/loose.py | 5 +++++ util.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/db/loose.py b/db/loose.py index c9ea3a038..a91f0d9d3 100644 --- a/db/loose.py +++ b/db/loose.py @@ -28,6 +28,7 @@ to_hex_sha, hex_to_bin, exists, + chmod, isdir, mkdir, rename, @@ -199,6 +200,10 @@ def store(self, istream): mkdir(obj_dir) # END handle destination directory rename(tmp_path, obj_path) + + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rrr + chmod(obj_path, 0444) # END handle dry_run istream.sha = hexsha diff --git a/util.py b/util.py index f0bf3e60f..2d9838379 100644 --- a/util.py +++ b/util.py @@ -54,6 +54,7 @@ def unpack_from(fmt, data, offset=0): # os shortcuts exists = os.path.exists mkdir = os.mkdir +chmod = os.chmod isdir = os.path.isdir rename = os.rename dirname = os.path.dirname @@ -291,6 +292,9 @@ def _end_writing(self, successful=True): # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + chmod(self._filepath, 0444) else: # just delete the file so far, we failed os.remove(lockfile) From e3d5ad195d9dfa46af3d931f9769e965e337daf7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 25 Jun 2010 15:29:50 +0200 Subject: [PATCH 0040/1790] CompoundDB: implemented simple dict base first-level cache which really helps to improve performance in real-world applications, which need to quickly determine whether objects are in or out for instance, as it happens during index_to_tree conversion Added separate pack streaming test which shows only a throughput of 250 streams / s in a densely packed pack, and about 3.5 MiB of data throughput. Performance tests show that half the time is spent in collecting the numerous deltas, the other one in decompressing and applying them Fixed broken performance tests --- db/base.py | 14 ++++++++++- db/git.py | 5 ++-- db/pack.py | 2 +- db/ref.py | 4 ++- exc.py | 7 +++++- test/performance/test_pack.py | 6 +++-- test/performance/test_pack_streaming.py | 33 +++++++++++++++++++++++++ test/performance/test_stream.py | 1 + 8 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 test/performance/test_pack_streaming.py diff --git a/db/base.py b/db/base.py index 35c20b7e1..0e81f0364 100644 --- a/db/base.py +++ b/db/base.py @@ -183,10 +183,13 @@ class CompoundDB(ObjectDBR, LazyMixin, CachingDB): Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" - def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super(CompoundDB, self)._set_cache_(attr) def _db_query(self, sha): """:return: database containing the given 20 or 40 byte sha @@ -194,8 +197,15 @@ def _db_query(self, sha): # most databases use binary representations, prevent converting # it everytime a database is being queried sha = to_bin_sha(sha) + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + for db in self._dbs: if db.has_object(sha): + self._db_cache[sha] = db return db # END for each database raise BadObject(sha) @@ -232,6 +242,8 @@ def databases(self): return tuple(self._dbs) def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() stat = False for db in self._dbs: if isinstance(db, CachingDB): diff --git a/db/git.py b/db/git.py index ad9a613b3..a9298df05 100644 --- a/db/git.py +++ b/db/git.py @@ -57,8 +57,9 @@ def _set_cache_(self, attr): # finally set the value self._loose_db = loose_db - - # END handle dbs + else: + super(GitDB, self)._set_cache_(attr) + # END handle attrs #{ ObjectDBW interface diff --git a/db/pack.py b/db/pack.py index af6f7ffd6..a78c4a8a0 100644 --- a/db/pack.py +++ b/db/pack.py @@ -46,7 +46,7 @@ def __init__(self, root_path): def _set_cache_(self, attr): if attr == '_entities': self._entities = list() - self.update_cache() + self.update_cache(force=True) # END handle entities initialization def _sort_entities(self): diff --git a/db/ref.py b/db/ref.py index 3a4813979..c149c03d0 100644 --- a/db/ref.py +++ b/db/ref.py @@ -21,7 +21,9 @@ def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() self._update_dbs_from_ref_file() - # END handle dbs + else: + super(ReferenceDB, self)._set_cache_(attr) + # END handle attrs def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls diff --git a/exc.py b/exc.py index 482726e3b..037ac3855 100644 --- a/exc.py +++ b/exc.py @@ -1,4 +1,5 @@ """Module with common exceptions""" +from util import to_hex_sha class ODBError(Exception): """All errors thrown by the object database""" @@ -7,7 +8,11 @@ class InvalidDBRoot(ODBError): """Thrown if an object database cannot be initialized at the given path""" class BadObject(ODBError): - """The object with the given SHA does not exist""" + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) class BadObjectType(ODBError): """The object had an unsupported type""" diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 8046c1f83..66101a35f 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -72,8 +72,10 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - - print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns + def _disabled_test_correctness(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" for crc in range(2): count = 0 st = time() diff --git a/test/performance/test_pack_streaming.py b/test/performance/test_pack_streaming.py new file mode 100644 index 000000000..4d47cdfcc --- /dev/null +++ b/test/performance/test_pack_streaming.py @@ -0,0 +1,33 @@ +"""Specific test for pack streams only""" +from lib import ( + TestBigRepoR + ) + +from gitdb.db.pack import PackedDB + +import os +import sys +from time import time + +class TestPackStreamingPerformance(TestBigRepoR): + + def test_stream_reading(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 5de463ee2..79fa9bc09 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -1,6 +1,7 @@ """Performance data streaming performance""" from lib import TestBigRepoR from gitdb.db import * +from gitdb.base import * from gitdb.stream import * from gitdb.util import pool from gitdb.typ import str_blob_type From 9e313a4773c97425d5c52be34ee21cbe405ddb84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 25 Jun 2010 16:39:25 +0200 Subject: [PATCH 0041/1790] gitdb now uses 20 byte shas internally only, reducing the need to convert shas around all the time, saving previous function calls, and memory after all --- base.py | 43 ++++++++++++++------------------- db/base.py | 17 ++++++------- db/loose.py | 18 +++++++------- db/mem.py | 6 ++--- db/pack.py | 6 +---- test/db/lib.py | 28 ++++++++++----------- test/db/test_git.py | 3 ++- test/db/test_ref.py | 15 +++++++----- test/performance/test_pack.py | 2 +- test/performance/test_stream.py | 13 ++++++---- test/test_base.py | 12 ++++----- test/test_pack.py | 14 +++++------ util.py | 1 + 13 files changed, 85 insertions(+), 93 deletions(-) diff --git a/base.py b/base.py index 0f4e63176..938a09242 100644 --- a/base.py +++ b/base.py @@ -1,7 +1,6 @@ """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( - to_hex_sha, - to_bin_sha, + bin_to_hex, zlib ) @@ -17,13 +16,13 @@ #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provdiing information - about the sha of the object, the type_string as well as the uncompressed size + """Carries information about an object in an ODB, provding information + about the binary sha of the object, the type_string as well as the uncompressed size in bytes. It can be accessed using tuple notation and using attribute access notation:: - assert dbi[0] == dbi.sha + assert dbi[0] == dbi.binsha assert dbi[1] == dbi.type assert dbi[2] == dbi.size @@ -38,18 +37,14 @@ def __init__(self, *args): #{ Interface @property - def sha(self): + def binsha(self): + """:return: our sha as binary, 20 bytes""" return self[0] - + @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) + return bin_to_hex(self[0]) @property def type(self): @@ -197,16 +192,10 @@ def __init__(self, type, size, stream, sha=None): list.__init__(self, (sha, type, size, stream, None)) #{ Interface - @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) + return bin_to_hex(self[0]) def _error(self): """:return: the error that occurred when processing the stream, or None""" @@ -231,13 +220,13 @@ def read(self, size=-1): #{ interface - def _set_sha(self, sha): - self[0] = sha + def _set_binsha(self, binsha): + self[0] = binsha - def _sha(self): + def _binsha(self): return self[0] - sha = property(_sha, _set_sha) + binsha = property(_binsha, _set_binsha) def _type(self): @@ -280,9 +269,13 @@ def __init__(self, sha, exc): tuple.__init__(self, (sha, exc)) @property - def sha(self): + def binsha(self): return self[0] + @property + def hexsha(self): + return bin_to_hex(self[0]) + @property def error(self): """:return: exception instance explaining the failure""" diff --git a/db/base.py b/db/base.py index 0e81f0364..c687166f7 100644 --- a/db/base.py +++ b/db/base.py @@ -2,8 +2,7 @@ from gitdb.util import ( pool, join, - LazyMixin, - to_bin_sha + LazyMixin ) from gitdb.exc import BadObject @@ -20,8 +19,7 @@ class ObjectDBR(object): """Defines an interface for object database lookup. - Objects are identified either by hex-sha (40 bytes) or - by sha (20 bytes)""" + Objects are identified either by their 20 byte bin sha""" def __contains__(self, sha): return self.has_obj @@ -29,14 +27,14 @@ def __contains__(self, sha): #{ Query Interface def has_object(self, sha): """ - :return: True if the object identified by the given 40 byte hexsha or 20 bytes + :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") def has_object_async(self, reader): """Return a reader yielding information about the membership of objects as identified by shas - :param reader: Reader yielding 20 byte or 40 byte shas. + :param reader: Reader yielding 20 byte shas. :return: async.Reader yielding tuples of (sha, bool) pairs which indicate whether the given sha exists in the database or not""" task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) @@ -44,7 +42,7 @@ def has_object_async(self, reader): def info(self, sha): """ :return: OInfo instance - :param sha: 40 bytes hexsha or 20 bytes binary sha + :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") @@ -57,7 +55,7 @@ def info_async(self, reader): def stream(self, sha): """:return: OStream instance - :param sha: 40 bytes hexsha or 20 bytes binary sha + :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") @@ -192,11 +190,10 @@ def _set_cache_(self, attr): super(CompoundDB, self)._set_cache_(attr) def _db_query(self, sha): - """:return: database containing the given 20 or 40 byte sha + """:return: database containing the given 20 byte sha :raise BadObject:""" # most databases use binary representations, prevent converting # it everytime a database is being queried - sha = to_bin_sha(sha) try: return self._db_cache[sha] except KeyError: diff --git a/db/loose.py b/db/loose.py index a91f0d9d3..7afc5bf1b 100644 --- a/db/loose.py +++ b/db/loose.py @@ -25,8 +25,8 @@ from gitdb.util import ( file_contents_ro_filepath, ENOENT, - to_hex_sha, hex_to_bin, + bin_to_hex, exists, chmod, isdir, @@ -100,7 +100,7 @@ def _map_loose_object(self, sha): """ :return: memory map of that file to allow random read access :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(to_hex_sha(sha))) + db_path = self.db_path(self.object_path(bin_to_hex(sha))) try: return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) except OSError,e: @@ -109,11 +109,11 @@ def _map_loose_object(self, sha): try: return file_contents_ro_filepath(db_path) except OSError: - raise BadObject(to_hex_sha(sha)) + raise BadObject(sha) # didn't work because of our flag, don't try it again self._fd_open_flags = 0 else: - raise BadObject(to_hex_sha(sha)) + raise BadObject(sha) # END handle error # END exception handling try: @@ -144,7 +144,7 @@ def stream(self, sha): def has_object(self, sha): try: - self.readable_db_object_path(to_hex_sha(sha)) + self.readable_db_object_path(bin_to_hex(sha)) return True except BadObject: return False @@ -158,7 +158,7 @@ def store(self, istream): # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - if istream.sha is None: + if istream.binsha is None: writer = FDCompressedSha1Writer(fd) else: writer = FDStream(fd) @@ -167,7 +167,7 @@ def store(self, istream): try: try: - if istream.sha is not None: + if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) @@ -187,7 +187,7 @@ def store(self, istream): # END assure tmpfile removal on error hexsha = None - if istream.sha: + if istream.binsha: hexsha = istream.hexsha else: hexsha = writer.sha(as_hex=True) @@ -206,7 +206,7 @@ def store(self, istream): chmod(obj_path, 0444) # END handle dry_run - istream.sha = hexsha + istream.binsha = hex_to_bin(hexsha) return istream def sha_iter(self): diff --git a/db/mem.py b/db/mem.py index 9e3d3972d..f361ab801 100644 --- a/db/mem.py +++ b/db/mem.py @@ -10,7 +10,6 @@ IStream, ) -from gitdb.util import to_bin_sha from gitdb.exc import ( BadObject, UnsupportedOperation @@ -54,7 +53,7 @@ def store(self, istream): # don't provide a size, the stream is written in object format, hence the # header needs decompression decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) - self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) return istream @@ -62,14 +61,13 @@ def store_async(self, reader): raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") def has_object(self, sha): - return to_bin_sha(sha) in self._cache + return sha in self._cache def info(self, sha): # we always return streams, which are infos as well return self.stream(sha) def stream(self, sha): - sha = to_bin_sha(sha) try: ostream = self._cache[sha] # rewind stream for the next one to read diff --git a/db/pack.py b/db/pack.py index a78c4a8a0..1a1c390ea 100644 --- a/db/pack.py +++ b/db/pack.py @@ -5,10 +5,7 @@ CachingDB ) -from gitdb.util import ( - to_bin_sha, - LazyMixin - ) +from gitdb.util import LazyMixin from gitdb.exc import ( BadObject, @@ -65,7 +62,6 @@ def _pack_info(self, sha): self._sort_entities() # END update sorting - sha = to_bin_sha(sha) for item in self._entities: index = item[2](sha) if index is not None: diff --git a/test/db/lib.py b/test/db/lib.py index b22f9a2b8..0080d919e 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -41,15 +41,15 @@ def _assert_object_writing_simple(self, db): istream = IStream(str_blob_type, len(data), StringIO(data)) new_istream = db.store(istream) assert new_istream is istream - assert db.has_object(istream.sha) + assert db.has_object(istream.binsha) - info = db.info(istream.sha) + info = db.info(istream.binsha) assert isinstance(info, OInfo) assert info.type == istream.type and info.size == istream.size - stream = db.stream(istream.sha) + stream = db.stream(istream.binsha) assert isinstance(stream, OStream) - assert stream.sha == info.sha and stream.type == info.type + assert stream.binsha == info.binsha and stream.type == info.type assert stream.read() == data # END for each item @@ -80,10 +80,10 @@ def _assert_object_writing(self, db): # store returns same istream instance, with new sha set my_istream = db.store(istream) - sha = istream.sha + sha = istream.binsha assert my_istream is istream assert db.has_object(sha) != dry_run - assert len(sha) == 40 # for now we require 40 byte shas as default + assert len(sha) == 20 # verify data - the slow way, we want to run code if not dry_run: @@ -107,12 +107,12 @@ def _assert_object_writing(self, db): # identical to what we fed in ostream.seek(0) istream.stream = ostream - assert istream.sha is not None - prev_sha = istream.sha + assert istream.binsha is not None + prev_sha = istream.binsha db.set_ostream(ZippedStoreShaWriter()) db.store(istream) - assert istream.sha == prev_sha + assert istream.binsha == prev_sha new_ostream = db.ostream() # note: only works as long our store write uses the same compression @@ -143,12 +143,12 @@ def istream_generator(offset=0, ni=ni): for stream in istreams: assert stream.error is None - assert len(stream.sha) == 40 + assert len(stream.binsha) == 20 assert isinstance(stream, IStream) # END assert each stream # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) hasobject_reader = db.has_object_async(reader) count = 0 for sha, has_object in hasobject_reader: @@ -158,7 +158,7 @@ def istream_generator(offset=0, ni=ni): assert count == ni # read the objects we have just written - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) ostream_reader = db.stream_async(reader) # read items individually to prevent hitting possible sys-limits @@ -171,7 +171,7 @@ def istream_generator(offset=0, ni=ni): assert count == ni # get info about our items - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) info_reader = db.info_async(reader) count = 0 @@ -186,7 +186,7 @@ def istream_generator(offset=0, ni=ni): # add 2500 items, and obtain their output streams nni = 2500 reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ] + istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] istream_reader = db.store_async(reader) istream_reader.set_post_cb(istream_to_sha) diff --git a/test/db/test_git.py b/test/db/test_git.py index 4d463f1f7..779e3f15e 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,6 +1,7 @@ from lib import * from gitdb.db import GitDB from gitdb.base import OStream, OInfo +from gitdb.util import hex_to_bin class TestGitDB(TestDBBase): @@ -11,7 +12,7 @@ def test_reading(self): assert 1 < len(gdb.databases()) < 4 # access should be possible - gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) assert gdb.size() > 200 diff --git a/test/db/test_ref.py b/test/db/test_ref.py index 68d9b8116..9df25cef6 100644 --- a/test/db/test_ref.py +++ b/test/db/test_ref.py @@ -1,6 +1,11 @@ from lib import * from gitdb.db import ReferenceDB - + +from gitdb.util import ( + NULL_BIN_SHA, + hex_to_bin + ) + import os class TestReferenceDB(TestDBBase): @@ -15,8 +20,7 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - null_sha_bin = '\0' * 20 - null_sha_hex = "0" * 40 + NULL_BIN_SHA = '\0' * 20 alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) @@ -25,8 +29,7 @@ def test_writing(self, path): assert len(list(rdb.sha_iter())) == 0 # try empty, non-existing - assert not rdb.has_object(null_sha_hex) - assert not rdb.has_object(null_sha_bin) + assert not rdb.has_object(NULL_BIN_SHA) # setup alternate file @@ -37,7 +40,7 @@ def test_writing(self, path): assert len(rdb.databases()) == 1 # we should now find a default revision of ours - gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert rdb.has_object(gitdb_sha) # remove valid diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 66101a35f..af468b0be 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -56,7 +56,7 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) # END for each function # retrieve stream and read all diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 79fa9bc09..1afc1a1a0 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -3,7 +3,10 @@ from gitdb.db import * from gitdb.base import * from gitdb.stream import * -from gitdb.util import pool +from gitdb.util import ( + pool, + bin_to_hex + ) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size @@ -73,10 +76,10 @@ def test_large_data_streaming(self, path): # writing - due to the compression it will seem faster than it is st = time() - sha = ldb.store(IStream('blob', size, stream)).sha + sha = ldb.store(IStream('blob', size, stream)).binsha elapsed_add = time() - st assert ldb.has_object(sha) - db_file = ldb.readable_db_object_path(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) fsize_kib = os.path.getsize(db_file) / 1000 @@ -151,7 +154,7 @@ def istream_iter(): # chunk size is not important as the stream will not really be decompressed # until its read - istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) + istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) ostream_reader = ldb.stream_async(istream_reader) chunk_task = TestStreamReader(ostream_reader, "chunker", None) @@ -172,7 +175,7 @@ def istream_iter(): istream_reader = ldb.store_async(reader) istream_reader.task().max_chunksize = 1 - istream_to_sha = lambda items: [ i.sha for i in items ] + istream_to_sha = lambda items: [ i.binsha for i in items ] istream_reader.set_post_cb(istream_to_sha) ostream_reader = ldb.stream_async(istream_reader) diff --git a/test/test_base.py b/test/test_base.py index 8d8bcc944..740e50bcd 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -7,7 +7,7 @@ from gitdb import * from gitdb.util import ( - NULL_HEX_SHA + NULL_BIN_SHA ) from gitdb.typ import ( @@ -19,12 +19,12 @@ class TestBaseTypes(TestBase): def test_streams(self): # test info - sha = NULL_HEX_SHA + sha = NULL_BIN_SHA s = 20 blob_id = 3 info = OInfo(sha, str_blob_type, s) - assert info.sha == sha + assert info.binsha == sha assert info.type == str_blob_type assert info.type_id == blob_id assert info.size == s @@ -72,9 +72,9 @@ def test_streams(self): # test istream istream = IStream(str_blob_type, s, stream) - assert istream.sha == None - istream.sha = sha - assert istream.sha == sha + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha assert len(istream.binsha) == 20 assert len(istream.hexsha) == 40 diff --git a/test/test_pack.py b/test/test_pack.py index 6821097ad..eaa2d38eb 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -134,8 +134,8 @@ def test_pack_entity(self): count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): count += 1 - assert info.sha == stream.sha - assert len(info.sha) == 20 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 assert info.type_id == stream.type_id assert info.size == stream.size @@ -143,17 +143,17 @@ def test_pack_entity(self): assert not info.type_id in delta_types # try all calls - assert len(entity.collect_streams(info.sha)) - assert isinstance(entity.info(info.sha), OInfo) - assert isinstance(entity.stream(info.sha), OStream) + assert len(entity.collect_streams(info.binsha)) + assert isinstance(entity.info(info.binsha), OInfo) + assert isinstance(entity.stream(info.binsha), OStream) # verify the stream try: - assert entity.is_valid_stream(info.sha, use_crc=True) + assert entity.is_valid_stream(info.binsha, use_crc=True) except UnsupportedOperation: pass # END ignore version issues - assert entity.is_valid_stream(info.sha, use_crc=False) + assert entity.is_valid_stream(info.binsha, use_crc=False) # END for each info, stream tuple assert count == size diff --git a/util.py b/util.py index 2d9838379..4d6f5b1e2 100644 --- a/util.py +++ b/util.py @@ -66,6 +66,7 @@ def unpack_from(fmt, data, offset=0): # constants NULL_HEX_SHA = "0"*40 +NULL_BIN_SHA = "\0"*20 #} END Aliases From c265c97f9130d2225b923b427736796c0a0d957c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 29 Jun 2010 10:56:50 +0200 Subject: [PATCH 0042/1790] Fixed critical bug that could cause single bytes not to be returned in reads that should read all --- stream.py | 4 +--- .../7b/b839852ed5e3a069966281bb08d50012fb309b | Bin 0 -> 446 bytes test/test_stream.py | 15 ++++++++++++--- util.py | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b diff --git a/stream.py b/stream.py index 3f5d0373a..010102bdd 100644 --- a/stream.py +++ b/stream.py @@ -249,9 +249,7 @@ def read(self, size=-1): # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data @@ -269,7 +267,7 @@ def read(self, size=-1): # Note: dcompdat can be empty even though we still appear to have bytes # to read, if we are called by compressed_bytes_read - it manipulates # us to empty the stream - if dcompdat and len(dcompdat) < size and self._br < self._s: + if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: dcompdat += self.read(size-len(dcompdat)) # END handle special case return dcompdat diff --git a/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b new file mode 100644 index 0000000000000000000000000000000000000000..021c2db3456031e1d2ab0bf73701fd07e1c3f18e GIT binary patch literal 446 zcmV;v0YUzF0V^p=O;s>8GGZ_^FfcPQQP4}zEJ-XWDauSLElDkA*k$BmEOsS@@9cup z94Ax`K0BwU`xdG)kwHqqdSOS!%y&8Fc7K0ZetX8c%q|BenU%5@~ z(-?H(E^K=G^sI4tWL0NrNXl|<-X9uJ$(;P;_{5YH25$lN^J%6=Iy|xceG!-L+2w=@ zF(E0*%}-&FY^mnnsdq=;+uk^0*18Z;q%))7P|&5i(pk;rewb_Um*b^Tbx?LaCci_=A6&R?hE*8 ztJ*Xl-{zzfCJ8maG_NQ%C$S_oB_0%@@wthac?_w(_A_#sd|uhiWEPKR`LO@V34K`5 opeZk|%uB|nyow>rIet2~&OwIvJHJh8HJG&J%aR?}0KQhvi8Wi_>i_@% literal 0 HcmV?d00001 diff --git a/test/test_stream.py b/test/test_stream.py index 859920719..dd65a1782 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -4,12 +4,14 @@ DummyStream, Sha1Writer, make_bytes, - make_object + make_object, + fixture_path ) from gitdb import * from gitdb.util import ( - NULL_HEX_SHA + NULL_HEX_SHA, + hex_to_bin ) from gitdb.util import zlib @@ -135,4 +137,11 @@ def test_compressed_writer(self): os.remove(path) # END for each os - + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + diff --git a/util.py b/util.py index 4d6f5b1e2..b55f789c0 100644 --- a/util.py +++ b/util.py @@ -56,6 +56,7 @@ def unpack_from(fmt, data, offset=0): mkdir = os.mkdir chmod = os.chmod isdir = os.path.isdir +isfile = os.path.isfile rename = os.rename dirname = os.path.dirname basename = os.path.basename @@ -288,7 +289,7 @@ def _end_writing(self, successful=True): if self._write and successful: # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": - if os.path.isfile(self._filepath): + if isfile(self._filepath): os.remove(self._filepath) # END remove if exists # END win32 special handling From 155b62a9af0aa7677078331e111d0f7aa6eb4afc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 00:05:38 +0200 Subject: [PATCH 0043/1790] Added empty version of gitdb documentation --- doc/Makefile | 89 ++++++++++++++++++++ doc/source/conf.py | 194 +++++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 20 +++++ 3 files changed, 303 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..b10926ae2 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GitDB.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GitDB.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..58b8791a1 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# GitDB documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 30 00:01:32 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'GitDB' +copyright = u'2010, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0' +# The full version, including alpha/beta/rc tags. +release = '1.0.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'GitDBdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'GitDB.tex', u'GitDB Documentation', + u'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..71ead1fc3 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,20 @@ +.. GitDB documentation master file, created by + sphinx-quickstart on Wed Jun 30 00:01:32 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GitDB's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + From 78e21e74a12f0767f6011a78349af52555ad2f74 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 11:22:05 +0200 Subject: [PATCH 0044/1790] Added auto-doc api reference and fixed plenty of docstrings --- db/base.py | 20 +++++-- db/pack.py | 4 +- doc/.gitignore | 1 + doc/source/api.rst | 113 ++++++++++++++++++++++++++++++++++++++++ doc/source/conf.py | 2 +- doc/source/index.rst | 4 ++ doc/source/intro.rst | 4 ++ doc/source/tutorial.rst | 4 ++ fun.py | 38 +++++++++----- pack.py | 39 +++++++++----- stream.py | 14 +++-- util.py | 18 ++++--- 12 files changed, 218 insertions(+), 43 deletions(-) create mode 100644 doc/.gitignore create mode 100644 doc/source/api.rst create mode 100644 doc/source/intro.rst create mode 100644 doc/source/tutorial.rst diff --git a/db/base.py b/db/base.py index c687166f7..02584c635 100644 --- a/db/base.py +++ b/db/base.py @@ -90,7 +90,9 @@ def __init__(self, *args, **kwargs): #{ Edit Interface def set_ostream(self, stream): - """Adjusts the stream to which all data should be sent when storing new objects + """ + Adjusts the stream to which all data should be sent when storing new objects + :param stream: if not None, the stream to use, if None the default stream will be used. :return: previously installed stream, or None if there was no override @@ -100,13 +102,16 @@ def set_ostream(self, stream): return cstream def ostream(self): - """:return: overridden output stream this instance will write to, or None + """ + :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream def store(self, istream): - """Create a new object in the database + """ + Create a new object in the database :return: the input istream object with its sha set to its corresponding value + :param istream: IStream compatible instance. If its sha is already set to a value, the object will just be stored in the our database format, in which case the input stream is expected to be in object format ( header + contents ). @@ -114,16 +119,19 @@ def store(self, istream): raise NotImplementedError("To be implemented in subclass") def store_async(self, reader): - """Create multiple new objects in the database asynchronously. The method will + """ + Create multiple new objects in the database asynchronously. The method will return right away, returning an output channel which receives the results as they are computed. :return: Channel yielding your IStream which served as input, in any order. The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. + :param reader: async.Reader yielding IStream instances. The same instances will be used in the output channel as were received in by the Reader. + :note:As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" @@ -167,8 +175,10 @@ class CachingDB(object): #{ Interface def update_cache(self, force=False): - """Call this method if the underlying data changed to trigger an update + """ + Call this method if the underlying data changed to trigger an update of the internal caching structures. + :param force: if True, the update must be performed. Otherwise the implementation may decide not to perform an update if it thinks nothing has changed. :return: True if an update was performed as something change indeed""" diff --git a/db/pack.py b/db/pack.py index 1a1c390ea..022efe0a2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -128,8 +128,10 @@ def store_async(self, reader): #{ Interface def update_cache(self, force=False): - """Update our cache with the acutally existing packs on disk. Add new ones, + """ + Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones + :param force: If True, the cache will be updated even though the directory does not appear to have changed according to its modification timestamp. :return: True if the packs have been updated so there is new information, diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..417671da9 --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,113 @@ +.. _api_reference_toplevel: + +############# +API Reference +############# + +**************** +Database.Base +**************** + +.. automodule:: gitdb.db.base + :members: + :undoc-members: + +**************** +Database.Git +**************** + +.. automodule:: gitdb.db.git + :members: + :undoc-members: + +**************** +Database.Loose +**************** + +.. automodule:: gitdb.db.loose + :members: + :undoc-members: + +**************** +Database.Memory +**************** + +.. automodule:: gitdb.db.mem + :members: + :undoc-members: + +**************** +Database.Pack +**************** + +.. automodule:: gitdb.db.pack + :members: + :undoc-members: + +****************** +Database.Reference +****************** + +.. automodule:: gitdb.db.ref + :members: + :undoc-members: + +************ +Base +************ + +.. automodule:: gitdb.base + :members: + :undoc-members: + +************ +Functions +************ + +.. automodule:: gitdb.fun + :members: + :undoc-members: + +************ +Pack +************ + +.. automodule:: gitdb.pack + :members: + :undoc-members: + +************ +Streams +************ + +.. automodule:: gitdb.stream + :members: + :undoc-members: + +************ +Types +************ + +.. automodule:: gitdb.typ + :members: + :undoc-members: + + +************ +Utilities +************ + +.. automodule:: gitdb.util + :members: + :undoc-members: + + + + + + + + + + + diff --git a/doc/source/conf.py b/doc/source/conf.py index 58b8791a1..d8aadabef 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../../')) # -- General configuration ----------------------------------------------------- diff --git a/doc/source/index.rst b/doc/source/index.rst index 71ead1fc3..409c78e99 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,6 +10,10 @@ Contents: .. toctree:: :maxdepth: 2 + + intro + tutorial + api Indices and tables ================== diff --git a/doc/source/intro.rst b/doc/source/intro.rst new file mode 100644 index 000000000..9565e766e --- /dev/null +++ b/doc/source/intro.rst @@ -0,0 +1,4 @@ +######## +Overview +######## + diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 000000000..b23a17703 --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,4 @@ + +######## +Tutorial +######## diff --git a/fun.py b/fun.py index d7e43717c..ccd8c0fc5 100644 --- a/fun.py +++ b/fun.py @@ -39,20 +39,22 @@ # used when dealing with larger streams chunk_size = 1000*mmap.PAGESIZE -__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', - 'write_object' ) +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data' ) #{ Routines def is_loose_object(m): - """:return: True the file contained in memory map m appears to be a loose object. - Only the first two bytes are needed""" + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" b0, b1 = map(ord, m[:2]) word = (b0 << 8) + b1 return b0 == 0x78 and (word % 31) == 0 def loose_object_header_info(m): - """:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well @@ -61,9 +63,10 @@ def loose_object_header_info(m): return type_name, int(size) def pack_object_header_info(data): - """:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) - The type_id should be interpreted according to the ``type_id_to_type_map`` map - The byte-offset specifies the start of the actual zlib compressed datastream + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" c = ord(data[0]) # first byte i = 1 # next char to read @@ -87,8 +90,9 @@ def pack_object_header_info(data): # END handle exceptions def msb_size(data, offset=0): - """:return: tuple(read_bytes, size) read the msb size from the given random - access data starting at the given byte offset""" + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" size = 0 i = 0 l = len(data) @@ -107,12 +111,14 @@ def msb_size(data, offset=0): return i+offset, size def loose_object_header(type, size): - """:return: string representing the loose object header, which is immediately + """ + :return: string representing the loose object header, which is immediately followed by the content stream of size 'size'""" return "%s %i\0" % (type, size) def write_object(type, size, read, write, chunk_size=chunk_size): - """Write the object as identified by type, size and source_stream into the + """ + Write the object as identified by type, size and source_stream into the target_stream :param type: type string of the object @@ -131,8 +137,10 @@ def write_object(type, size, read, write, chunk_size=chunk_size): return tbw def stream_copy(read, write, size, chunk_size): - """Copy a stream up to size bytes using the provided read and write methods, + """ + Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size + :note: its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written @@ -156,8 +164,10 @@ def stream_copy(read, write, size, chunk_size): def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): - """Apply data from a delta buffer using a source buffer to the target file, + """ + Apply data from a delta buffer using a source buffer to the target file, which will be written to + :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes diff --git a/pack.py b/pack.py index c66d6717e..ed873b269 100644 --- a/pack.py +++ b/pack.py @@ -383,8 +383,9 @@ def version(self): return self._version def data(self): - """:return: read-only data of this pack. It provides random access and usually - is a memory map""" + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map""" return self._data def checksum(self): @@ -428,19 +429,22 @@ def collect_streams(self, offset): def info(self, offset): """Retrieve information about the object at the given file-absolute offset + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._data, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._data, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): - """:return: iterator yielding OPackStream compatible instances, allowing - to access the data in the pack directly. + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. :note: Iterating a pack directly is costly as the datastream has to be decompressed @@ -555,6 +559,7 @@ def _object(self, sha, as_stream, index=-1): def info(self, sha): """Retrieve information about the object identified by the given sha + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" @@ -562,6 +567,7 @@ def info(self, sha): def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha + :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" @@ -589,15 +595,18 @@ def index(self): return self._index def is_valid_stream(self, sha, use_crc=False): - """Verify that the stream at the given sha is valid. - :param sha: 20 byte sha1 of the object whose stream to verify + """ + Verify that the stream at the given sha is valid. + :param use_crc: if True, the index' crc for the sha is used to determine - whether the compressed stream of the object is valid. If it is + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the data of the actual undeltified object, as it depends on more than just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -639,18 +648,22 @@ def is_valid_stream(self, sha, use_crc=False): return True def info_iter(self): - """:return: Iterator over all objects in this pack. The iterator yields + """ + :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) def stream_iter(self): - """:return: iterator over all objects in this pack. The iterator yields - OStream instances""" + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" return self._iter_objects(as_stream=True) def collect_streams_at_offset(self, offset): - """As the version in the PackFile, but can resolve REF deltas within this pack + """ + As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) @@ -678,11 +691,13 @@ def collect_streams_at_offset(self, offset): return streams def collect_streams(self, sha): - """As ``PackFile.collect_streams``, but takes a sha instead of an offset. + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. Additionally, ref_delta streams will be resolved within this pack. If this is not possible, the stream will be left alone, hence it is adivsed to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. diff --git a/stream.py b/stream.py index 010102bdd..675ccf27e 100644 --- a/stream.py +++ b/stream.py @@ -77,6 +77,7 @@ def __del__(self): def _parse_header_info(self): """If this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content + :return: parsed type_string, size""" # read header maxb = 512 # should really be enough, cgit uses 8192 I believe @@ -105,6 +106,7 @@ def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. + :param m: memory map on which to oparate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" @@ -117,8 +119,9 @@ def data(self): return self._m def compressed_bytes_read(self): - """:return: number of compressed bytes read. This includes the bytes it - took to decompress the header ( if there was one )""" + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" # ABSTRACT: When decompressing a byte stream, it can be that the first # x bytes which were requested match the first x bytes in the loosely # compressed datastream. This is the worst-case assumption that the reader @@ -406,6 +409,7 @@ def read(self, count=0): def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != os.SEEK_SET: raise ValueError("Can only seek to position 0") @@ -417,11 +421,14 @@ def seek(self, offset, whence=os.SEEK_SET): @classmethod def new(cls, stream_list): - """Convert the given list of streams into a stream which resolves deltas + """ + Convert the given list of streams into a stream which resolves deltas when reading from it. + :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handled""" @@ -526,6 +533,7 @@ def getvalue(self): class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor + :note: operates on raw file descriptors :note: for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") diff --git a/util.py b/util.py index b55f789c0..502ac94dd 100644 --- a/util.py +++ b/util.py @@ -154,25 +154,26 @@ def to_bin_sha(sha): class LazyMixin(object): """ - Base class providing an interface to lazily retrieve attribute values upon + Base class providing an interface to lazily retrieve attribute values upon first access. If slots are used, memory will only be reserved once the attribute - is actually accessed and retrieved the first time. All future accesses will + is actually accessed and retrieved the first time. All future accesses will return the cached value as stored in the Instance's dict or slot. """ + __slots__ = tuple() def __getattr__(self, attr): """ Whenever an attribute is requested that we do not know, we allow it to be created and set. Next time the same attribute is reqeusted, it is simply - returned from our dict/slots. - """ + returned from our dict/slots. """ self._set_cache_(attr) # will raise in case the cache was not created return object.__getattribute__(self, attr) def _set_cache_(self, attr): - """ This method should be overridden in the derived class. + """ + This method should be overridden in the derived class. It should check whether the attribute named by attr can be created and cached. Do nothing if you do not know the attribute or call your subclass @@ -183,7 +184,8 @@ def _set_cache_(self, attr): class LockedFD(object): - """This class facilitates a safe read and write operation to a file on disk. + """ + This class facilitates a safe read and write operation to a file on disk. If we write to 'file', we obtain a lock file at 'file.lock' and write to that instead. If we succeed, the lock file will be renamed to overwrite the original file. @@ -212,7 +214,9 @@ def _lockfilepath(self): return "%s.lock" % self._filepath def open(self, write=False, stream=False): - """Open the file descriptor for reading or writing, both in binary mode. + """ + Open the file descriptor for reading or writing, both in binary mode. + :param write: if True, the file descriptor will be opened for writing. Other wise it will be opened read-only. :param stream: if True, the file descriptor will be wrapped into a simple stream From 4cde3c046b71bf481418cd3a2a0f0bf5bc540a2b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 21:52:12 +0200 Subject: [PATCH 0045/1790] Added a minimal documentation, including a quick usage guide --- doc/source/api.rst | 2 +- doc/source/conf.py | 11 ++-- doc/source/intro.rst | 25 +++++++++ doc/source/tutorial.rst | 116 ++++++++++++++++++++++++++++++++++++++-- test/test_example.py | 53 ++++++++++++++++++ 5 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 test/test_example.py diff --git a/doc/source/api.rst b/doc/source/api.rst index 417671da9..9dea95764 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -1,4 +1,4 @@ -.. _api_reference_toplevel: +.. _api-label: ############# API Reference diff --git a/doc/source/conf.py b/doc/source/conf.py index d8aadabef..8e2585c2f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.0' +version = '0.5' # The full version, including alpha/beta/rc tags. -release = '1.0.0' +release = '0.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -93,10 +93,9 @@ # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} +html_theme_options = { + "stickysidebar": "true" +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 9565e766e..a51d8bff4 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -2,3 +2,28 @@ Overview ######## +The *GitDB* project implements interfaces to allow read and write access to git repositories. In its core lies the *db* package, which contains all database types necessary to read a complete git repository. These are the ``LooseObjectDB``, the ``PackedDB`` and the ``ReferenceDB`` which are combined into the ``GitDB`` to combine every aspect of the git database. + +For this to work, GitDB implements pack reading, as well as loose object reading and writing. Data is always encapsulated in streams, which allows huge files to be handled as well as small ones, usually only chunks of the stream are kept in memory for processing, never the whole stream at once. + +Interfaces are used to describe the API, making it easy to provide alternate implementations. + +================ +Installing GitDB +================ +Its easiest to install gitdb using the *easy_install* program, which is part of the `setuptools`_:: + + $ easy_install gitdb + +As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +=============== +Getting Started +=============== +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index b23a17703..cfe3fb284 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -1,4 +1,114 @@ +.. _tutorial-label: -######## -Tutorial -######## +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying types. + +****** +Design +****** +The *GitDB* project models a standard git object database and implements it in pure python. This means that data, being classified by one of four types, can can be stored in the database and will in future be referred to by the generated SHA1 key, which is a 20 byte string within python. + +*GitDB* implements *RW* access to loose objects, as well as *RO* access to packed objects. Compound Databases allow to combine multiple object databases into one. + +All data is read and written using streams, which effectively prevents more than a chunk of the data being kept in memory at once mostly [#]_. + +******* +Streams +******* +In order to assure the object database can handle objects of any size, a stream interface is used for data retrieval as well as to fill data into the database. + +Basic Stream Types +================== +There are two fundamentally different types of streams, **IStream**\ s and **OStream**\ s. IStreams are mutable and are used to provide data streams to the database to create new objects. + +OStreams are immutable and are used to read data from the database. The base of this type, **OInfo**, contains only type and size information of the queried object, but no stream, which is slightly faster to retrieve depending on the database. + +OStreams are tuples, IStreams are lists. Both, OInfo and OStream, have the same member ordering which allows quick conversion from one type to another. + +**************************** +Data Query and Data Addition +**************************** +Databases support query and/or addition of objects using simple interfaces. They are called **ObjectDBR** for read-only access, and **ObjectDBW** for write access to create new objects. + +Both have two sets of methods, one of which allows interacting with single objects, the other one allowing to handle a stream of objects simultaneously and asynchronously. + +Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: + + + ldb = LooseObjectDB(fixture_path("../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + # END for each sha in database + +To store information, you prepare an *IStream* object with the required information. The provided stream will be read and converted into an object, and the respective 20 byte SHA1 identifier is stored in the IStream object:: + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + +********************** +Asynchronous Operation +********************** +For each read or write method that allows a single-object to be handled, an *_async* version exists which reads items to be processed from a channel, and writes the operation's result into an output channel that is read by the caller or by other async methods, to support chaining. + +Using asynchronous operations is easy, but chaining multiple operations together to form a complex one would require you to read the docs of the *async* package. At the current time, due to the *GIL*, the *GitDB* can only achieve true concurrency during zlib compression and decompression if big objects, if the respective c modules where compiled in *async*. + +Asynchronous operations are scheduled by a *ThreadPool* which resides in the *gitdb.util* module:: + + from gitdb.util import pool + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) + + +Use async methods with readers, which supply items to be processed. The result is given through readers as well:: + + from async import IteratorReader + + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + + +********* +Databases +********* +A database implements different interfaces, one if which will always be the *ObjectDBR* interface to support reading of object information and streams. + +The *Loose Object Database* as well as the *Packed Object Database* are *File Databases*, hence they operate on a directory which contains files they can read. + +File databases implementing the *ObjectDBW* interface can also be forced to write their output into the specified stream, using the ``set_ostream`` method. This effectively allows you to redirect its output to anywhere you like. + +*Compound Databases* are not implementing their own access type, but instead combine multiple database implementations into one. Examples for this database type are the *Reference Database*, which reads object locations from a file, and the *GitDB* which combines loose, packed and referenced objects into one database interface. + +For more information about the individual database types, please see the :ref:`API Reference `, and the unittests for the respective types. + + +---- + +.. [#] When reading streams from packs, all deltas are currently applied and the result written into a memory map before the first byte is returned. Future versions of the delta-apply algorithm might improve on this. diff --git a/test/test_example.py b/test/test_example.py new file mode 100644 index 000000000..dc3d6230a --- /dev/null +++ b/test/test_example.py @@ -0,0 +1,53 @@ +"""Module with examples from the tutorial section of the docs""" +from lib import * +from gitdb import IStream +from gitdb.db import LooseObjectDB +from gitdb.util import pool + +from cStringIO import StringIO + +from async import IteratorReader + +class TestExamples(TestBase): + + def test_base(self): + ldb = LooseObjectDB(fixture_path("../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + + + # async operation + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) From 7562fdd96ab995f6c25fc102ef40a285283c844e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jul 2010 13:30:14 +0200 Subject: [PATCH 0046/1790] added setup.py and Manifest to allow distribution and easy_install --- .gitignore | 3 +++ MANIFEST.in | 15 +++++++++++++++ README | 2 +- doc/source/intro.rst | 12 ++++++++++++ ext/async | 2 +- setup.py | 18 ++++++++++++++++++ 6 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 MANIFEST.in create mode 100755 setup.py diff --git a/.gitignore b/.gitignore index 1a9d961a7..1e097f6f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +MANIFEST +build/ +dist/ *.pyc *.o *.so diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..a01acc452 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +include VERSION +include LICENSE +include CHANGES +include AUTHORS +include README + +include _fun.c + +graft test + +global-exclude .git* +global-exclude *.pyc +global-exclude *.so +global-exclude *.dll +global-exclude *.o diff --git a/README b/README index a52d9f508..33a566d86 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -GtDB +GitDB ===== GitDB allows you to access bare git repositories for reading and writing. It diff --git a/doc/source/intro.rst b/doc/source/intro.rst index a51d8bff4..4d675cbc4 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -26,4 +26,16 @@ Getting Started =============== It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. +================= +Source Repository +================= +The latest source can be cloned using git from one of the following locations: + + * git://gitorious.org/git-python/gitdb.git + * git://github.com/Byron/gitdb.git + +License Information +=================== +*GitDB* is licensed under the New BSD License. + .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/ext/async b/ext/async index 796b5e94f..a18235276 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 796b5e94f19dfc36a3fb251468192373c76510b0 +Subproject commit a1823527631ffa8a2438c95096e71a741df7b61e diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..b6c2c414b --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +from distutils.core import setup, Extension + +setup(name = "gitdb", + version = "0.5.0", + description = "Git Object Database", + author = "Sebastian Thiel", + author_email = "byronimo@gmail.com", + url = "http://gitorious.org/git-python/gitdb", + packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), + package_data={'gitdb' : ['AUTHORS', 'README'], + 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + package_dir = {'gitdb':''}, + ext_modules=[Extension('gitdb._fun', ['_fun.c'])], + license = "BSD License", + requires=('async (>=0.6.0)',), + long_description = """GitDB is a pure-Python git object database""" + ) From b76bac22bbb146a7a82318721147d7a5f831b7e9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jul 2010 18:23:53 +0200 Subject: [PATCH 0047/1790] Removed obsolete makefile, to compile the extension, use ./setup build_ext and copy _fun.so from the build directory into the root directory --- makefile | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 makefile diff --git a/makefile b/makefile deleted file mode 100644 index 390289625..000000000 --- a/makefile +++ /dev/null @@ -1,12 +0,0 @@ - -_fun.o: _fun.c - gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@ - -_fun.so: _fun.o - gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@ - -all: _fun.so - -clean: - -rm *.so - -rm *.o From 6c8721a7d5d32e54bb4ffd3725ed23ac5d76a593 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 2 Jul 2010 16:22:40 +0200 Subject: [PATCH 0048/1790] Win32 compatability adjustments. One performance test fails during shutdown as python fails to delete a stream in time it seems, so the file cannot be deleted as it is still opened by someone. This is madness, raising the question how badly python truly fails to call the destructors of objects in order to allow them to release their resources --- db/loose.py | 16 ++++++++++++++-- test/lib.py | 6 ++++++ test/test_example.py | 3 +++ test/test_stream.py | 5 ++++- util.py | 8 ++++++-- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/db/loose.py b/db/loose.py index 7afc5bf1b..86eb151fb 100644 --- a/db/loose.py +++ b/db/loose.py @@ -30,6 +30,8 @@ exists, chmod, isdir, + isfile, + remove, mkdir, rename, dirname, @@ -60,6 +62,12 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): # chunks in which data will be copied between streams stream_chunk_size = chunk_size + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = 0444 + if os.name == 'nt': + new_objects_mode = 0644 + def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) @@ -199,11 +207,15 @@ def store(self, istream): if not isdir(obj_dir): mkdir(obj_dir) # END handle destination directory + # rename onto existing doesn't work on windows + if os.name == 'nt' and isfile(obj_path): + remove(obj_path) + # END handle win322 rename(tmp_path, obj_path) # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rrr - chmod(obj_path, 0444) + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) # END handle dry_run istream.binsha = hex_to_bin(hexsha) diff --git a/test/lib.py b/test/lib.py index 742aa7f5c..3fb87d547 100644 --- a/test/lib.py +++ b/test/lib.py @@ -19,6 +19,7 @@ import tempfile import shutil import os +import gc #{ Bases @@ -44,6 +45,11 @@ def wrapper(self): print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) raise finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + gc.collect() shutil.rmtree(path) # END handle exception # END wrapper diff --git a/test/test_example.py b/test/test_example.py index dc3d6230a..dc82436ec 100644 --- a/test/test_example.py +++ b/test/test_example.py @@ -21,6 +21,9 @@ def test_base(self): assert len(ostream.read()) == ostream.size assert ldb.has_object(oinfo.binsha) # END for each sha in database + # assure we close all files + del(ostream) + del(oinfo) data = "my data" istream = IStream("blob", len(data), StringIO(data)) diff --git a/test/test_stream.py b/test/test_stream.py index dd65a1782..948cbe766 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -19,6 +19,7 @@ str_blob_type ) +import time import tempfile import os @@ -125,12 +126,14 @@ def test_compressed_writer(self): # for now, just a single write, code doesn't care about chunking assert len(data) == ostream.write(data) ostream.close() + # its closed already self.failUnlessRaises(OSError, os.close, fd) # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY) + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) os.close(fd) assert written_data == zlib.compress(data, 1) # best speed diff --git a/util.py b/util.py index 502ac94dd..6b2f91073 100644 --- a/util.py +++ b/util.py @@ -58,12 +58,14 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir isfile = os.path.isfile rename = os.rename +remove = os.remove dirname = os.path.dirname basename = os.path.basename join = os.path.join read = os.read write = os.write close = os.close +fsync = os.fsync # constants NULL_HEX_SHA = "0"*40 @@ -128,7 +130,7 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): :note: for now we don't try to use O_NOATIME directly as the right value needs to be shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|flags) + fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: return file_contents_ro(fd, stream, allow_mmap) finally: @@ -300,7 +302,9 @@ def _end_writing(self, successful=True): os.rename(lockfile, self._filepath) # assure others can at least read the file - the tmpfile left it at rw-- - chmod(self._filepath, 0444) + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, 0644) else: # just delete the file so far, we failed os.remove(lockfile) From 46bf4710e0f7184ac4875e8037de30b5081bfda2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jul 2010 00:34:41 +0200 Subject: [PATCH 0049/1790] PackEntity: fixed capital bug which would cause a None to be in place of the bin sha when querying infos or streams through an entity --- ext/async | 2 +- pack.py | 4 +++- test/test_pack.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/async b/ext/async index a18235276..76f15fc4b 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit a1823527631ffa8a2438c95096e71a741df7b61e +Subproject commit 76f15fc4b3e3ccb0160d6c887181f29095d16f23 diff --git a/pack.py b/pack.py index ed873b269..5b13bcc56 100644 --- a/pack.py +++ b/pack.py @@ -516,6 +516,9 @@ def _object(self, sha, as_stream, index=-1): # its a little bit redundant here, but it needs to be efficient if index < 0: index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) offset = self._index.offset(index) type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) if as_stream: @@ -551,7 +554,6 @@ def _object(self, sha, as_stream, index=-1): # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) # END handle stream diff --git a/test/test_pack.py b/test/test_pack.py index eaa2d38eb..445ed4a17 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -144,8 +144,12 @@ def test_pack_entity(self): # try all calls assert len(entity.collect_streams(info.binsha)) - assert isinstance(entity.info(info.binsha), OInfo) - assert isinstance(entity.stream(info.binsha), OStream) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None # verify the stream try: From e750ce0be7d3cbf694c095f6519e2e34b2c3332c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 10:40:35 +0200 Subject: [PATCH 0050/1790] Added method to obtain a full sha from a partial sha, for each of the databases. The IndexFile has a new method to retrieve an index from a partial sha, instead of from a full sha, to provide the required functionality --- db/git.py | 65 ++++++++++++++++++++++++++++++++++++++++++- db/loose.py | 20 ++++++++++++- db/pack.py | 22 +++++++++++++++ exc.py | 6 +++- pack.py | 48 +++++++++++++++++++++++++++++++- test/db/test_git.py | 13 +++++++-- test/db/test_loose.py | 14 +++++++++- test/db/test_pack.py | 24 ++++++++++++++++ test/test_pack.py | 6 ++++ 9 files changed, 211 insertions(+), 7 deletions(-) diff --git a/db/git.py b/db/git.py index a9298df05..62fed7fb6 100644 --- a/db/git.py +++ b/db/git.py @@ -9,11 +9,33 @@ from ref import ReferenceDB from gitdb.util import LazyMixin -from gitdb.exc import InvalidDBRoot +from gitdb.exc import ( + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) import os +from gitdb.util import hex_to_bin + __all__ = ('GitDB', ) + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + + class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" @@ -71,4 +93,45 @@ def ostream(self): def set_ostream(self, ostream): return self._loose_db.set_ostream(ostream) + #} END objectdbw interface + + #{ Interface + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + if len(partial_hexsha) % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if isinstance(db, LooseObjectDB): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface diff --git a/db/loose.py b/db/loose.py index 86eb151fb..521be44c2 100644 --- a/db/loose.py +++ b/db/loose.py @@ -7,7 +7,8 @@ from gitdb.exc import ( InvalidDBRoot, - BadObject, + BadObject, + AmbiguousObjectName ) from gitdb.stream import ( @@ -102,6 +103,23 @@ def readable_db_object_path(self, hexsha): # END handle cache raise BadObject(hexsha) + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(partial_hexsha): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + #} END interface def _map_loose_object(self, sha): diff --git a/db/pack.py b/db/pack.py index 022efe0a2..47d511378 100644 --- a/db/pack.py +++ b/db/pack.py @@ -10,6 +10,7 @@ from gitdb.exc import ( BadObject, UnsupportedOperation, + AmbiguousObjectName ) from gitdb.pack import PackEntity @@ -175,5 +176,26 @@ def update_cache(self, force=False): def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] + + def partial_to_complete_sha(self, partial_binsha): + """:return: 20 byte sha as inferred by the given partial binary sha + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) #} END interface diff --git a/exc.py b/exc.py index 037ac3855..012cdbc6b 100644 --- a/exc.py +++ b/exc.py @@ -13,7 +13,11 @@ class BadObject(ODBError): def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) - + +class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" + class BadObjectType(ODBError): """The object had an unsupported type""" diff --git a/pack.py b/pack.py index 5b13bcc56..ab7c9b677 100644 --- a/pack.py +++ b/pack.py @@ -302,7 +302,53 @@ def sha_to_index(self, sha): # END handle midpoint # END bisect return None - + + def partial_sha_to_index(self, partial_sha): + """:return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_sha: an at least two bytes of a partial sha + :raise AmbiguousObjectName:""" + if len(partial_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + first_byte = ord(partial_sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + len_partial = len(partial_sha) + # fill the partial to full 20 bytes + filled_sha = partial_sha + '\0'*(20 - len_partial) + + # find lowest + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(filled_sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + if lo < self.size: + cur_sha = get_sha(lo) + if cur_sha[:len_partial] == partial_sha: + next_sha = None + if lo+1 < self.size: + next_sha = get_sha(lo+1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_sha) + return lo + # END if we have a match + # END if we found something + return None + if 'PackIndexFile_sha_to_index' in globals(): # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses diff --git a/test/db/test_git.py b/test/db/test_git.py index 779e3f15e..1c0c39c96 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,7 +1,8 @@ from lib import * +from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo -from gitdb.util import hex_to_bin +from gitdb.util import hex_to_bin, bin_to_hex class TestGitDB(TestDBBase): @@ -16,7 +17,15 @@ def test_reading(self): assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) assert gdb.size() > 200 - assert len(list(gdb.sha_iter())) == gdb.size() + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + + # test partial shas + for binsha in sha_list: + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8]) == binsha + # END for each sha + + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") @with_rw_directory def test_writing(self, path): diff --git a/test/db/test_loose.py b/test/db/test_loose.py index 536b02048..8e8a9bfc3 100644 --- a/test/db/test_loose.py +++ b/test/db/test_loose.py @@ -1,10 +1,12 @@ from lib import * from gitdb.db import LooseObjectDB +from gitdb.exc import BadObject +from gitdb.util import bin_to_hex class TestLooseDB(TestDBBase): @with_rw_directory - def test_writing(self, path): + def test_basics(self, path): ldb = LooseObjectDB(path) # write data @@ -16,3 +18,13 @@ def test_writing(self, path): assert shas and len(shas[0]) == 20 assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be foudn + diff --git a/test/db/test_pack.py b/test/db/test_pack.py index f347f408b..c8974b29b 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -2,6 +2,8 @@ from gitdb.db import PackedDB from gitdb.test.lib import fixture_path +from gitdb.exc import BadObject, AmbiguousObjectName + import os import random @@ -44,3 +46,25 @@ def test_writing(self, path): info = pdb.info(sha) stream = pdb.stream(sha) # END for each sha to query + + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambigious ... + # assert num_ambiguous + + # non-existing + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0") diff --git a/test/test_pack.py b/test/test_pack.py index 445ed4a17..a9db19178 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -57,7 +57,13 @@ def _assert_index_file(self, index, version, size): assert entry[0] == index.offset(oidx) assert entry[1] == sha assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4,8,11,17,20): + assert index.partial_sha_to_index(sha[:l]) == oidx + # END for each object index in indexfile + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0") def _assert_pack_file(self, pack, version, size): From e9e0496ea982f7574c6ea379e9f6e85dd111fc8e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 11:10:25 +0200 Subject: [PATCH 0051/1790] Moved partial_to_complete_sha_hex from gitdb to compounddb, its fitting there better --- db/base.py | 61 +++++++++++++++++++++++++++++++++++++++++++-- db/git.py | 56 ----------------------------------------- test/db/test_git.py | 11 ++++++-- 3 files changed, 68 insertions(+), 60 deletions(-) diff --git a/db/base.py b/db/base.py index 02584c635..ebd5040a5 100644 --- a/db/base.py +++ b/db/base.py @@ -2,10 +2,14 @@ from gitdb.util import ( pool, join, - LazyMixin + LazyMixin, + hex_to_bin ) -from gitdb.exc import BadObject +from gitdb.exc import ( + BadObject, + AmbiguousObjectName + ) from async import ( ChannelThreadTask @@ -186,6 +190,22 @@ def update_cache(self, force=False): # END interface + + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): """A database which delegates calls to sub-databases. @@ -258,6 +278,43 @@ def update_cache(self, force=False): # END if is caching db # END for each database to update return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + if len(partial_hexsha) % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + #} END interface diff --git a/db/git.py b/db/git.py index 62fed7fb6..f0e63b15a 100644 --- a/db/git.py +++ b/db/git.py @@ -16,26 +16,9 @@ ) import os -from gitdb.util import hex_to_bin - __all__ = ('GitDB', ) -def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed - and compound databases.""" - if isinstance(database, CompoundDB): - compounds = list() - dbs = database.databases() - output.extend(db for db in dbs if not isinstance(db, CompoundDB)) - for cdb in (db for db in dbs if isinstance(db, CompoundDB)): - _databases_recursive(cdb, output) - else: - output.append(database) - # END handle database type - - - class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" @@ -96,42 +79,3 @@ def set_ostream(self, ostream): #} END objectdbw interface - #{ Interface - - def partial_to_complete_sha_hex(self, partial_hexsha): - """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha - :param partial_hexsha: hexsha with less than 40 byte - :raise AmbiguousObjectName: """ - databases = list() - _databases_recursive(self, databases) - - if len(partial_hexsha) % 2 != 0: - partial_binsha = hex_to_bin(partial_hexsha + "0") - else: - partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - - candidate = None - for db in databases: - full_bin_sha = None - try: - if isinstance(db, LooseObjectDB): - full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) - else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha) - # END handle database type - except BadObject: - continue - # END ignore bad objects - if full_bin_sha: - if candidate and candidate != full_bin_sha: - raise AmbiguousObjectName(partial_hexsha) - candidate = full_bin_sha - # END handle candidate - # END for each db - if not candidate: - raise BadObject(partial_binsha) - return candidate - - #} END interface diff --git a/test/db/test_git.py b/test/db/test_git.py index 1c0c39c96..d2ae10bad 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -20,9 +20,16 @@ def test_reading(self): sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module # test partial shas - for binsha in sha_list: - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8]) == binsha + # this one as uneven and quite short + assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha # END for each sha self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") From ac7d4757ab4041f5f0f5806934130024b098bb82 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 12:10:49 +0200 Subject: [PATCH 0052/1790] Fixed bug caused by incorrect comparison of 'filled-up' binary shas, as it would compare even the filling, instead of taking the canonical length into account and hence ignore the last 4 bytes if the original hex sha had an odd length --- db/base.py | 5 +++-- db/pack.py | 8 ++++++-- fun.py | 21 ++++++++++++++++++++- pack.py | 28 ++++++++++++++++------------ test/db/test_pack.py | 4 ++-- test/test_pack.py | 4 ++-- 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/db/base.py b/db/base.py index ebd5040a5..1914dbbce 100644 --- a/db/base.py +++ b/db/base.py @@ -287,7 +287,8 @@ def partial_to_complete_sha_hex(self, partial_hexsha): databases = list() _databases_recursive(self, databases) - if len(partial_hexsha) % 2 != 0: + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") else: partial_binsha = hex_to_bin(partial_hexsha) @@ -300,7 +301,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if hasattr(db, 'partial_to_complete_sha_hex'): full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha) + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) # END handle database type except BadObject: continue diff --git a/db/pack.py b/db/pack.py index 47d511378..0ec8a4e3b 100644 --- a/db/pack.py +++ b/db/pack.py @@ -177,13 +177,17 @@ def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - def partial_to_complete_sha(self, partial_binsha): + def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for item in self._entities: - item_index = item[1].index().partial_sha_to_index(partial_binsha) + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) if item_index is not None: sha = item[1].index().sha(item_index) if candidate and candidate != sha: diff --git a/fun.py b/fun.py index ccd8c0fc5..e8bc35531 100644 --- a/fun.py +++ b/fun.py @@ -40,7 +40,8 @@ chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data' ) + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha' ) #{ Routines @@ -223,5 +224,23 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" + +def is_equal_canonical_sha(canonical_length, match, sha1): + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length/2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + #} END routines diff --git a/pack.py b/pack.py index ab7c9b677..6966c295f 100644 --- a/pack.py +++ b/pack.py @@ -12,6 +12,7 @@ from fun import ( pack_object_header_info, + is_equal_canonical_sha, type_id_to_type_map, write_object, stream_copy, @@ -303,24 +304,26 @@ def sha_to_index(self, sha): # END bisect return None - def partial_sha_to_index(self, partial_sha): - """:return: index as in `sha_to_index` or None if the sha was not found in this - index file - :param partial_sha: an at least two bytes of a partial sha + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha + :param canonical_length: lenght of the original hexadecimal representation of the + given partial binary sha :raise AmbiguousObjectName:""" - if len(partial_sha) < 2: + if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - first_byte = ord(partial_sha[0]) + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - len_partial = len(partial_sha) # fill the partial to full 20 bytes - filled_sha = partial_sha + '\0'*(20 - len_partial) + filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) # find lowest while lo < hi: @@ -336,14 +339,15 @@ def partial_sha_to_index(self, partial_sha): lo = mid + 1 # END handle midpoint # END bisect - if lo < self.size: + + if lo < self.size(): cur_sha = get_sha(lo) - if cur_sha[:len_partial] == partial_sha: + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size: + if lo+1 < self.size(): next_sha = get_sha(lo+1) if next_sha and next_sha == cur_sha: - raise AmbiguousObjectName(partial_sha) + raise AmbiguousObjectName(partial_bin_sha) return lo # END if we have a match # END if we found something diff --git a/test/db/test_pack.py b/test/db/test_pack.py index c8974b29b..0386b3f80 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -55,7 +55,7 @@ def test_writing(self, path): for i, sha in enumerate(sha_list): short_sha = sha[:max((i % max_bytes), min_bytes)] try: - assert pdb.partial_to_complete_sha(short_sha) == sha + assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha except AmbiguousObjectName: num_ambiguous += 1 pass # valid, we can have short objects @@ -67,4 +67,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0") + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/test/test_pack.py b/test/test_pack.py index a9db19178..717d07e46 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -60,10 +60,10 @@ def _assert_index_file(self, index, version, size): # verify partial sha for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l]) == oidx + assert index.partial_sha_to_index(sha[:l], l*2) == oidx # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0") + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) def _assert_pack_file(self, pack, version, size): From f534e6e9a24f2ac7e7e0f3679551b512d4af569a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jul 2010 11:24:20 +0200 Subject: [PATCH 0053/1790] Added fixes to setup.py to allow easy_installation --- ext/async | 2 +- setup.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/ext/async b/ext/async index 76f15fc4b..081978422 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 76f15fc4b3e3ccb0160d6c887181f29095d16f23 +Subproject commit 0819784229dc98f92d2c57d740c9aebd533846d6 diff --git a/setup.py b/setup.py index b6c2c414b..1afeb3e17 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,60 @@ #!/usr/bin/env python from distutils.core import setup, Extension - +from distutils.command.build_py import build_py + +import os, sys + +# wow, this is a mixed bag ... I am pretty upset about all of this ... +setuptools_build_py_module = None +try: + # don't pull it in if we don't have to + if 'setuptools' in sys.modules: + import setuptools.command.build_py as setuptools_build_py_module +except ImportError: + pass + +def get_data_files(self): + """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, + the line dealing with the ``plen`` has a bug which causes it to truncate too much. + It is fixed in the system interpreters as they receive patches, and shows how + bad it is if something doesn't have proper unittests. + The code here is a plain copy of the python2.6 version which works for all. + + Generate list of '(package,src_dir,build_dir,filenames)' tuples""" + data = [] + if not self.packages: + return data + + # this one is just for the setup tools ! They don't iniitlialize this variable + # when they should, but do it on demand using this method.Its crazy + if hasattr(self, 'analyze_manifest'): + self.analyze_manifest() + # END handle setuptools ... + + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir)+1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + +build_py.get_data_files = get_data_files +if setuptools_build_py_module: + setuptools_build_py_module.build_py._get_data_files = get_data_files +# END apply setuptools patch too + setup(name = "gitdb", version = "0.5.0", description = "Git Object Database", @@ -14,5 +68,6 @@ ext_modules=[Extension('gitdb._fun', ['_fun.c'])], license = "BSD License", requires=('async (>=0.6.0)',), + install_requires='async >= 0.6.0', long_description = """GitDB is a pure-Python git object database""" ) From ebf1ead5fb593bc559a96581b60f05165967d35d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 9 Jul 2010 11:14:47 +0200 Subject: [PATCH 0054/1790] stream: adjusted code to allow compilation in python 2.4. It doesn't work though as the mmap implementation isn't advanced enough, but parent projects can at least import gitdb, and possibly use different implementations on demand --- ext/async | 2 +- stream.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/async b/ext/async index 081978422..2842d1eae 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 0819784229dc98f92d2c57d740c9aebd533846d6 +Subproject commit 2842d1eaee8411f7d09c6dc533465b2a404740ec diff --git a/stream.py b/stream.py index 675ccf27e..90bfc996d 100644 --- a/stream.py +++ b/stream.py @@ -167,7 +167,7 @@ def compressed_bytes_read(self): #} END interface - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != os.SEEK_SET: @@ -407,7 +407,7 @@ def read(self, count=0): self._br += len(data) return data - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" @@ -517,7 +517,7 @@ def write(self, data): def close(self): self.buf.write(self.zip.flush()) - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" if offset != 0 or whence != os.SEEK_SET: From 18152febd428e67b86bb4fb68ec1691d4de75a9c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 9 Jul 2010 11:18:06 +0200 Subject: [PATCH 0055/1790] Bumped version to 0.5.1, added changelog to documentation --- doc/source/changes.rst | 12 ++++++++++++ doc/source/conf.py | 2 +- doc/source/index.rst | 1 + setup.py | 6 +++--- stream.py | 6 +++--- util.py | 43 ++++++++++++++++++++++++++++++++++++++---- 6 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 doc/source/changes.rst diff --git a/doc/source/changes.rst b/doc/source/changes.rst new file mode 100644 index 000000000..84738ae05 --- /dev/null +++ b/doc/source/changes.rst @@ -0,0 +1,12 @@ +######### +Changelog +######### +***** +0.5.1 +***** +* Restored most basic python 2.4 compatibility, such that gitdb can be imported within python 2.4, pack access cannot work though. This at least allows Super-Projects to provide their own workarounds, or use everything but pack support. + +***** +0.5.0 +***** +Initial Release diff --git a/doc/source/conf.py b/doc/source/conf.py index 8e2585c2f..e10addb8d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.0' +release = '0.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst index 409c78e99..d414cb22f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: intro tutorial api + changes Indices and tables ================== diff --git a/setup.py b/setup.py index 1afeb3e17..73d39d09f 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ def get_data_files(self): # END apply setuptools patch too setup(name = "gitdb", - version = "0.5.0", + version = "0.5.1", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", @@ -67,7 +67,7 @@ def get_data_files(self): package_dir = {'gitdb':''}, ext_modules=[Extension('gitdb._fun', ['_fun.c'])], license = "BSD License", - requires=('async (>=0.6.0)',), - install_requires='async >= 0.6.0', + requires=('async (>=0.6.1)',), + install_requires='async >= 0.6.1', long_description = """GitDB is a pure-Python git object database""" ) diff --git a/stream.py b/stream.py index 90bfc996d..3e55c83a4 100644 --- a/stream.py +++ b/stream.py @@ -170,7 +170,7 @@ def compressed_bytes_read(self): def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset @@ -411,7 +411,7 @@ def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self._br = 0 @@ -520,7 +520,7 @@ def close(self): def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self.buf.seek(0) diff --git a/util.py b/util.py index 6b2f91073..3316c24f7 100644 --- a/util.py +++ b/util.py @@ -3,7 +3,14 @@ import mmap import sys import errno -import cStringIO + +from cStringIO import StringIO + +# in py 2.4, StringIO is only StringI, without write support. +# Hence we must use the python implementation for this +if sys.version_info[1] < 5: + from StringIO import StringIO +# END handle python 2.4 try: import async.mod.zlib as zlib @@ -73,6 +80,29 @@ def unpack_from(fmt, data, offset=0): #} END Aliases +#{ compatibility stuff ... + +class _RandomAccessStringIO(object): + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = StringIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + +#} END compatibility stuff ... #{ Routines @@ -94,7 +124,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return cStringIO.StringIO("\0"*size) + return _RandomAccessStringIO("\0"*size) # END handle memory allocation @@ -109,7 +139,12 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): try: if allow_mmap: # supports stream and random access - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except EnvironmentError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 except OSError: pass # END exception handling @@ -117,7 +152,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: - return cStringIO.StringIO(contents) + return _RandomAccessStringIO(contents) return contents def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): From 449e80bf5f04c9ccc3b1b8926b833227d5fe2d5b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 13 Jul 2010 10:06:27 +0200 Subject: [PATCH 0056/1790] byteswapping in offsets method will only be done if required - previously it was always performed even though the byte order of the system didn't require it --- pack.py | 5 ++++- test/test_example.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pack.py b/pack.py index 6966c295f..91323fcce 100644 --- a/pack.py +++ b/pack.py @@ -52,6 +52,8 @@ from itertools import izip import array import os +import sys + __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -272,7 +274,8 @@ def offsets(self): a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more - a.byteswap() + if sys.byteorder == 'little': + a.byteswap() return a else: return tuple(self.offset(index) for index in xrange(self.size())) diff --git a/test/test_example.py b/test/test_example.py index dc82436ec..3fc3fcf5e 100644 --- a/test/test_example.py +++ b/test/test_example.py @@ -22,9 +22,13 @@ def test_base(self): assert ldb.has_object(oinfo.binsha) # END for each sha in database # assure we close all files - del(ostream) - del(oinfo) - + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + data = "my data" istream = IStream("blob", len(data), StringIO(data)) From 425ecf04aa5038c3d46b01ca20de17c51ef6c4e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 13 Jul 2010 10:52:11 +0200 Subject: [PATCH 0057/1790] Adjusted setup.py to deal more gracefully with build failures of our optional extensions --- setup.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 73d39d09f..265156df2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from distutils.core import setup, Extension from distutils.command.build_py import build_py +from distutils.command.build_ext import build_ext import os, sys @@ -10,9 +11,19 @@ # don't pull it in if we don't have to if 'setuptools' in sys.modules: import setuptools.command.build_py as setuptools_build_py_module + from setuptools.command.build_ext import build_ext except ImportError: pass +class build_ext_nofail(build_ext): + """Doesn't fail when build our optional extensions""" + def run(self): + try: + build_ext.run(self) + except Exception: + print "Ignored failure when building extensions, pure python modules will be used instead" + # END ignore errors + def get_data_files(self): """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, the line dealing with the ``plen`` has a bug which causes it to truncate too much. @@ -55,7 +66,8 @@ def get_data_files(self): setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too -setup(name = "gitdb", +setup(cmdclass={'build_ext':build_ext_nofail}, + name = "gitdb", version = "0.5.1", description = "Git Object Database", author = "Sebastian Thiel", From 274c5c89ba973b0ae7ee56424b160417f33d1d5e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Oct 2010 18:56:53 +0200 Subject: [PATCH 0058/1790] Added frame for actual implementation of the aggregation - for now we just implement a forward-merge of the chunks, then the reverse version once there is some more knowledge on how this works --- ext/async | 2 +- fun.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- stream.py | 26 +++++++++++++++-- util.py | 4 +++ 4 files changed, 109 insertions(+), 6 deletions(-) diff --git a/ext/async b/ext/async index 2842d1eae..5992bb6c8 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 2842d1eaee8411f7d09c6dc533465b2a404740ec +Subproject commit 5992bb6c85973ed81c54c71fef42e2413cd29e88 diff --git a/fun.py b/fun.py index e8bc35531..f18d567ac 100644 --- a/fun.py +++ b/fun.py @@ -41,7 +41,48 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha' ) + 'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas', + 'merge_deltas') + + +#{ Structures + +class DeltaChunk(object): + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data' # chunk of bytes to be added to the target buffer or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data + + #{ Interface + + def abssize(self): + return self.to + self.ts + + def apply(self, source, target): + """Apply own data to the target buffer + :param source: buffer providing source bytes for copy operations + :param target: target buffer large enough to contain all the changes to be applied""" + if self.data is not None: + # APPEND DATA + pass + else: + # COPY DATA FROM SOURCE + pass + # END handle chunk mode + + #} END interface + +#} END structures #{ Routines @@ -164,10 +205,46 @@ def stream_copy(read, write, size, chunk_size): return dbw +def reverse_merge_deltas(dcl, dstreams): + """Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + during the merge process + :param dstreams: iterable of delta stream objects. They must be ordered latest first, + hence the delta to be applied last comes first, then its ancestors + :return: None""" + raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") + +def merge_deltas(dcl, dstreams): + """Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + during the merge process + :param dstreams: iterable of delta stream objects. They must be ordered latest last, + hence the delta to be applied last comes last, its oldest ancestor first + :return: None""" + for ds in dstreams: + buf = ds.read() + i, src_size = msb_size(buf) + i, target_size = msb_size(buf, i) + + # parse the commands + + # END for each delta stream + +def apply_delta_chunks(src_buf, src_buf_size, dcl, target): + """ + Apply data from a delta chunk list and a source buffer to the target stream + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param target: ostream with a write method""" + + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): """ - Apply data from a delta buffer using a source buffer to the target file, - which will be written to + Apply data from a delta buffer using a source buffer to the target file :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes diff --git a/stream.py b/stream.py index 3e55c83a4..74dc2b3c4 100644 --- a/stream.py +++ b/stream.py @@ -7,7 +7,9 @@ from fun import ( msb_size, stream_copy, - apply_delta_data, + apply_delta_data, + apply_delta_chunks, + merge_deltas, delta_types ) @@ -320,9 +322,29 @@ def __init__(self, stream_list): self._br = 0 def _set_cache_(self, attr): + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + dcl = list() + reverse_merge_deltas(dcl, self._dstreams) + + if len(dcl) == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl[-1].abssize() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target) + + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" - # prefetch information buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: diff --git a/util.py b/util.py index 3316c24f7..1ea182025 100644 --- a/util.py +++ b/util.py @@ -117,6 +117,10 @@ def make_sha(source=''): def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessStringIO('') + # END handle empty chunks gracefully + try: return mmap.mmap(-1, size) # read-write by default except EnvironmentError: From afefae6de6fee561842c1a09069f9593b6bd62aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Oct 2010 22:46:51 +0200 Subject: [PATCH 0059/1790] Implemented everything around the actual merge-algorithm, which appears to be the heart of the whole thing --- fun.py | 236 +++++++++++++++++++++++++++++++++++++++++++++++------- stream.py | 14 ++-- 2 files changed, 214 insertions(+), 36 deletions(-) diff --git a/fun.py b/fun.py index f18d567ac..c2cbed504 100644 --- a/fun.py +++ b/fun.py @@ -41,12 +41,42 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas', - 'merge_deltas') + 'is_equal_canonical_sha', 'reverse_merge_deltas', + 'merge_deltas', 'DeltaChunkList') #{ Structures +def _trunc_delta(d, size): + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size""" + if size == 0: + raise ValueError("size to truncate to must not be 0") + if d.ts == size: + return + if size > d.ts: + raise ValueError("Cannot truncate delta 'larger'") + + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE, see _split_delta + +def _move_delta_offset(d, bytes): + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size""" + if bytes >= d.ts: + raise ValueError("Cannot move offset that much") + + d.to += bytes + d.ts -= bytes + if d.data: + d.data = d.data[bytes:] + # END handle data + + class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -68,20 +98,120 @@ def __init__(self, to, ts, so, data): def abssize(self): return self.to + self.ts - def apply(self, source, target): + def apply(self, source, write): """Apply own data to the target buffer :param source: buffer providing source bytes for copy operations - :param target: target buffer large enough to contain all the changes to be applied""" - if self.data is not None: - # APPEND DATA - pass - else: + :param write: write method to call with data to write""" + if self.data is None: # COPY DATA FROM SOURCE - pass + write(buffer(source, self.so, self.ts)) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + if self.ts < len(self.data): + write(self.data[:self.ts]) + else: + write(self.data) + # END handle truncation # END handle chunk mode #} END interface +def _closest_index(dcl, absofs): + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs + :note: global method for performance only, it belongs to DeltaChunkList""" + # TODO: binary search !! + for i,d in enumerate(dcl): + if absofs >= d.to: + return i + # END for each delta absofs + raise AssertionError("Should never be here") + +def _split_delta(dcl, absofs, di=None): + """Split the delta at di into two deltas, adjusting their sizes, absofss and data + accordingly and adding them to the dcl. + :param absofs: absolute absofs at which to split the delta + :param di: a pre-determined delta-index, or None if it should be retrieved + :note: it will not split if it + :return: the closest index which has been split ( usually di if given) + :note: belongs to DeltaChunkList""" + if di is None: + di = _closest_index(dcl, absofs) + + d = dcl[di] + if d.to == absofs or d.abssize() == absofs: + return di + + _trunc_delta(d, absofs - d.to) + + # insert new one + ds = d.abssize() + relsize = absofs - ds + + self.insert(di+1, DeltaChunk( ds, + relsize, + (d.so and ds) or None, + (d.data and d.data[relsize:]) or None)) + # END adjust next one + return di + +def _merge_delta(dcl, d): + """Merge the given DeltaChunk instance into the dcl""" + index = _closest_index(dcl, d.to) + od = dcl[index] + + if d.data is None: + if od.data: + # OVERWRITE DATA + pass + else: + # MERGE SOURCE AREA + pass + # END overwrite data + else: + if od.data: + # MERGE DATA WITH DATA + pass + else: + # INSERT DATA INTO COPY AREA + pass + # END combine or insert data + # END handle chunk mode + + +class DeltaChunkList(list): + """List with special functionality to deal with DeltaChunks""" + + def init(self, size): + """Intialize this instance with chunks defining to fill up size from a base + buffer of equal size""" + if len(self) != 0: + return + # pretend we have one huge delta chunk, which just copies everything + # from source to destination + maxint32 = 2**32 + for x in range(0, size, maxint32): + self.append(DeltaChunk(x, maxint32, x, None)) + # END create copy chunks + offset = x*maxint32 + remainder = size-offset + if remainder: + self.append(DeltaChunk(offset, remainder, offset, None)) + # END handle all done in loop + + def terminate_at(self, size): + """Chops the list at the given size, splitting and removing DeltaNodes + as required""" + di = _closest_index(self, size) + d = self[di] + rsize = size - d.to + if rsize: + _trunc_delta(d, rsize) + # END truncate last node if possible + del(self[di+(rsize!=0):]) + #} END structures #{ Routines @@ -204,12 +334,10 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw - def reverse_merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed - during the merge process + :param dcl: see merge_deltas :param dstreams: iterable of delta stream objects. They must be ordered latest first, hence the delta to be applied last comes first, then its ancestors :return: None""" @@ -218,31 +346,78 @@ def reverse_merge_deltas(dcl, dstreams): def merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + :param dcl: DeltaChunkList, may be empty initially, and will be changed during the merge process :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first :return: None""" for ds in dstreams: - buf = ds.read() - i, src_size = msb_size(buf) - i, target_size = msb_size(buf, i) + db = ds.read() + delta_buf_size = ds.size + + # read header + i, src_size = msb_size(db) + i, target_size = msb_size(db, i) - # parse the commands + if len(dcl) == 0: + dcl.init(target_size) + # END handle empty list + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_size): + break + + _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # TODO: Concatenate multiple deltachunks + _merge_delta(dcl, DeltaChunk(tbw, c, None, db[i:i+c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.terminate_at(target_size) # END for each delta stream -def apply_delta_chunks(src_buf, src_buf_size, dcl, target): - """ - Apply data from a delta chunk list and a source buffer to the target stream - - :param src_buf: random access data from which the delta was created - :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes - :param target: ostream with a write method""" - -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file @@ -250,10 +425,9 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data - :param target_file: file like object to write the result to + :param write: write method taking a chunk of bytes :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 - twrite = target_file.write db = delta_buf while i < delta_buf_size: c = ord(db[i]) @@ -289,9 +463,9 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if (rbound < cp_size or rbound > src_buf_size): break - twrite(buffer(src_buf, cp_off, cp_size)) + write(buffer(src_buf, cp_off, cp_size)) elif c: - twrite(db[i:i+c]) + write(db[i:i+c]) i += c else: raise ValueError("unexpected delta opcode 0") diff --git a/stream.py b/stream.py index 74dc2b3c4..9e507c83b 100644 --- a/stream.py +++ b/stream.py @@ -8,8 +8,8 @@ msb_size, stream_copy, apply_delta_data, - apply_delta_chunks, merge_deltas, + DeltaChunkList, delta_types ) @@ -325,8 +325,8 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = list() - reverse_merge_deltas(dcl, self._dstreams) + dcl = DeltaChunkList() + merge_deltas(dcl, self._dstreams) if len(dcl) == 0: self._size = 0 @@ -338,9 +338,13 @@ def _set_cache_(self, attr): self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target) + # APPLY CHUNKS + write = self._mm_target.write + for dc in dcl: + dc.apply(bbuf, write) + # END for each deltachunk to apply def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" From 4d41a878bf0f9171d9a1a44ef885a58c6a025e0d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 01:07:38 +0200 Subject: [PATCH 0060/1790] Initial version of the merge algorithm - the current implementation will lead to quite some fragementation, but that can be improved on once it works --- fun.py | 131 +++++++++++++++++++++++++++++++++++++++++------------- stream.py | 4 +- 2 files changed, 103 insertions(+), 32 deletions(-) diff --git a/fun.py b/fun.py index c2cbed504..2b1419cbb 100644 --- a/fun.py +++ b/fun.py @@ -47,7 +47,7 @@ #{ Structures -def _trunc_delta(d, size): +def _set_delta_rbound(d, size): """Truncate the given delta to the given size :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size""" @@ -63,7 +63,7 @@ def _trunc_delta(d, size): # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE, see _split_delta -def _move_delta_offset(d, bytes): +def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static :param bytes: amount of bytes to move, must be smaller than delta size""" @@ -71,6 +71,7 @@ def _move_delta_offset(d, bytes): raise ValueError("Cannot move offset that much") d.to += bytes + d.so += bytes d.ts -= bytes if d.data: d.data = d.data[bytes:] @@ -95,7 +96,7 @@ def __init__(self, to, ts, so, data): #{ Interface - def abssize(self): + def rbound(self): return self.to + self.ts def apply(self, source, write): @@ -129,39 +130,35 @@ def _closest_index(dcl, absofs): # END for each delta absofs raise AssertionError("Should never be here") -def _split_delta(dcl, absofs, di=None): - """Split the delta at di into two deltas, adjusting their sizes, absofss and data - accordingly and adding them to the dcl. - :param absofs: absolute absofs at which to split the delta - :param di: a pre-determined delta-index, or None if it should be retrieved - :note: it will not split if it - :return: the closest index which has been split ( usually di if given) +def _split_delta(dcl, d, di, relofs, insert_offset=0): + """Split the delta at di into two deltas, adjusting their sizes, offsets and data + accordingly and adding the new part to the dcl + :param relofs: relative offset at which to split the delta + :param d: delta chunk to split + :param di: index of d in dcl + :param insert_offset: offset for the new split id + :return: newly created DeltaChunk :note: belongs to DeltaChunkList""" - if di is None: - di = _closest_index(dcl, absofs) - - d = dcl[di] - if d.to == absofs or d.abssize() == absofs: - return di + if relofs > d.ts: + raise ValueError("Cannot split behinds a chunks rbound") - _trunc_delta(d, absofs - d.to) + osize = d.ts - relofs + _set_delta_rbound(d, relofs) # insert new one - ds = d.abssize() - relsize = absofs - ds + drb = d.rbound() - self.insert(di+1, DeltaChunk( ds, - relsize, - (d.so and ds) or None, - (d.data and d.data[relsize:]) or None)) - # END adjust next one - return di + nd = DeltaChunk( drb, + osize, + (d.so and d.so + osize) or None, + (d.data and d.data[osize:]) or None ) -def _merge_delta(dcl, d): - """Merge the given DeltaChunk instance into the dcl""" - index = _closest_index(dcl, d.to) - od = dcl[index] + self.insert(di+1+insert_offset, nd) + return nd +def _handle_merge(ld, rd): + """Optimize the layout of the lhs delta and the rhs delta + TODO: Once the default implementation is working""" if d.data is None: if od.data: # OVERWRITE DATA @@ -173,6 +170,7 @@ def _merge_delta(dcl, d): else: if od.data: # MERGE DATA WITH DATA + # overwrite the data at the respective spot pass else: # INSERT DATA INTO COPY AREA @@ -180,6 +178,79 @@ def _merge_delta(dcl, d): # END combine or insert data # END handle chunk mode +def _merge_delta(dcl, d): + """Merge the given DeltaChunk instance into the dcl + :param d: the DeltaChunk to merge""" + cdi = _closest_index(dcl, d.to) # current delta index + cd = dcl[cdi] # current delta + + # either we go at his spot, or after + # cdi either moves one up, or stays + dcl.insert(di + (d.to > cd.to), d) + cdi += d.to == cd.to + + while True: + # are we larger than the current block + if d.to < cd.to: + if d.rbound() >= cd.rbound(): + # xxx|xxx|x + # remove the current item completely + dcl.pop(cdi) + cdi -= 1 + elif d.rbound() > cd.to: + # MOVE ITS LBOUND + # xxx|x--| + _move_delta_lbound(cd, d.rbound() - cd.to) + break + else: + # WE DON'T OVERLAP IT + # this can possibly happen + assert False, "Wow, this can really happen" + break + # END rbound overlap handling + # END lbound overlap handling + else: + if d.to >= cd.rbound(): + #|---|...xx + break + # END + + if d.rbound() >= cd.rbound(): + if d.to == cd.to: + #|xxx|x + # REMOVE CD + dcl.pop(cdi) + cdi -= 1 + else: + # TRUNCATE CD + #|-xx| + _set_delta_rbound(cd, d.to - cd.to) + # END handle offset special case + elif d.to == cd.to: + #|x--| + # we shift it by our size + _move_delta_lbound(cd, d.ts) + else: + #|-x-| + # SPLIT CD AND LBOUND MOVE ITS SECOND PART + # insert offset is required to insert it after us + nd = _split_delta(dcl, cd, cdi, 1) + _move_delta_lbound(nd, d.ts) + break + # END handle rbound overlap + # END handle overlap + + cdi += 1 + if cdi < len(dcl): + cd = dcl[cdi] + else: + break + # END check for end of list + # while our chunk is not completely done + + + + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" @@ -208,7 +279,7 @@ def terminate_at(self, size): d = self[di] rsize = size - d.to if rsize: - _trunc_delta(d, rsize) + _set_delta_rbound(d, rsize) # END truncate last node if possible del(self[di+(rsize!=0):]) diff --git a/stream.py b/stream.py index 9e507c83b..3248ae464 100644 --- a/stream.py +++ b/stream.py @@ -326,7 +326,7 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = DeltaChunkList() - merge_deltas(dcl, self._dstreams) + merge_deltas(dcl, reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -334,7 +334,7 @@ def _set_cache_(self, attr): return # END handle empty list - self._size = dcl[-1].abssize() + self._size = dcl[-1].rbound() self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) From 48fdcf44c536e9ca1008749cf6cd4547303ca519 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 12:02:15 +0200 Subject: [PATCH 0061/1790] Added new pack to test database to get some ascii deltas, which may possibly help visual debugging ( but probably not ) added plenty of debug code, to realize that the copy operations are not yet correct --- fun.py | 120 +++++++++++------- stream.py | 15 ++- ...bf8e71d8c18879e499335762dd95119d93d9f1.idx | Bin 0 -> 2248 bytes ...f8e71d8c18879e499335762dd95119d93d9f1.pack | Bin 0 -> 3732 bytes test/test_pack.py | 7 +- 5 files changed, 93 insertions(+), 49 deletions(-) create mode 100644 test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx create mode 100644 test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack diff --git a/fun.py b/fun.py index 2b1419cbb..2d51f62eb 100644 --- a/fun.py +++ b/fun.py @@ -10,6 +10,7 @@ decompressobj = zlib.decompressobj import mmap +from itertools import islice, izip # INVARIANTS OFS_DELTA = 6 @@ -93,7 +94,10 @@ def __init__(self, to, ts, so, data): self.ts = ts self.so = so self.data = data - + + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + #{ Interface def rbound(self): @@ -105,6 +109,7 @@ def apply(self, source, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE + assert len(source) - self.so - self.ts > 0 write(buffer(source, self.so, self.ts)) else: # APPEND DATA @@ -121,14 +126,16 @@ def apply(self, source, write): def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs + absofs. :note: global method for performance only, it belongs to DeltaChunkList""" # TODO: binary search !! for i,d in enumerate(dcl): - if absofs >= d.to: + if absofs < d.to: + return i-1 + elif absofs == d.to: return i # END for each delta absofs - raise AssertionError("Should never be here") + return len(dcl)-1 def _split_delta(dcl, d, di, relofs, insert_offset=0): """Split the delta at di into two deltas, adjusting their sizes, offsets and data @@ -150,7 +157,7 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): nd = DeltaChunk( drb, osize, - (d.so and d.so + osize) or None, + d.so + osize, (d.data and d.data[osize:]) or None ) self.insert(di+1+insert_offset, nd) @@ -178,45 +185,51 @@ def _handle_merge(ld, rd): # END combine or insert data # END handle chunk mode -def _merge_delta(dcl, d): +def _merge_delta(dcl, dc): """Merge the given DeltaChunk instance into the dcl :param d: the DeltaChunk to merge""" - cdi = _closest_index(dcl, d.to) # current delta index + if len(dcl) == 0: + dcl.append(dc) + return + # END early return on empty list + + cdi = _closest_index(dcl, dc.to) # current delta index cd = dcl[cdi] # current delta # either we go at his spot, or after # cdi either moves one up, or stays - dcl.insert(di + (d.to > cd.to), d) - cdi += d.to == cd.to + #print "insert at %i" % (cdi + (dc.to > cd.to)) + #print cd, dc + dcl.insert(cdi + (dc.to > cd.to), dc) + cdi += dc.to == cd.to while True: # are we larger than the current block - if d.to < cd.to: - if d.rbound() >= cd.rbound(): + if dc.to < cd.to: + if dc.rbound() >= cd.rbound(): # xxx|xxx|x # remove the current item completely dcl.pop(cdi) cdi -= 1 - elif d.rbound() > cd.to: + elif dc.rbound() > cd.to: # MOVE ITS LBOUND # xxx|x--| - _move_delta_lbound(cd, d.rbound() - cd.to) + _move_delta_lbound(cd, dc.rbound() - cd.to) break else: # WE DON'T OVERLAP IT - # this can possibly happen - assert False, "Wow, this can really happen" + # this can actually happen, once multiple streams are merged break # END rbound overlap handling # END lbound overlap handling else: - if d.to >= cd.rbound(): + if dc.to >= cd.rbound(): #|---|...xx break # END - if d.rbound() >= cd.rbound(): - if d.to == cd.to: + if dc.rbound() >= cd.rbound(): + if dc.to == cd.to: #|xxx|x # REMOVE CD dcl.pop(cdi) @@ -224,18 +237,18 @@ def _merge_delta(dcl, d): else: # TRUNCATE CD #|-xx| - _set_delta_rbound(cd, d.to - cd.to) + _set_delta_rbound(cd, dc.to - cd.to) # END handle offset special case - elif d.to == cd.to: + elif dc.to == cd.to: #|x--| # we shift it by our size - _move_delta_lbound(cd, d.ts) + _move_delta_lbound(cd, dc.ts) else: #|-x-| # SPLIT CD AND LBOUND MOVE ITS SECOND PART # insert offset is required to insert it after us nd = _split_delta(dcl, cd, cdi, 1) - _move_delta_lbound(nd, d.ts) + _move_delta_lbound(nd, dc.ts) break # END handle rbound overlap # END handle overlap @@ -248,30 +261,14 @@ def _merge_delta(dcl, d): # END check for end of list # while our chunk is not completely done - + ## DEBUG ## + dcl.check_integrity() class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def init(self, size): - """Intialize this instance with chunks defining to fill up size from a base - buffer of equal size""" - if len(self) != 0: - return - # pretend we have one huge delta chunk, which just copies everything - # from source to destination - maxint32 = 2**32 - for x in range(0, size, maxint32): - self.append(DeltaChunk(x, maxint32, x, None)) - # END create copy chunks - offset = x*maxint32 - remainder = size-offset - if remainder: - self.append(DeltaChunk(offset, remainder, offset, None)) - # END handle all done in loop - def terminate_at(self, size): """Chops the list at the given size, splitting and removing DeltaNodes as required""" @@ -283,6 +280,38 @@ def terminate_at(self, size): # END truncate last node if possible del(self[di+(rsize!=0):]) + ## DEBUG ## + self.check_integrity(size) + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doen't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + if dc.data: + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self)-1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft,rgt in izip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + #} END structures #{ Routines @@ -422,7 +451,8 @@ def merge_deltas(dcl, dstreams): :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first :return: None""" - for ds in dstreams: + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi db = ds.read() delta_buf_size = ds.size @@ -430,10 +460,6 @@ def merge_deltas(dcl, dstreams): i, src_size = msb_size(db) i, target_size = msb_size(db, i) - if len(dcl) == 0: - dcl.init(target_size) - # END handle empty list - # interpret opcodes tbw = 0 # amount of target bytes written while i < delta_buf_size: @@ -475,7 +501,7 @@ def merge_deltas(dcl, dstreams): tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - _merge_delta(dcl, DeltaChunk(tbw, c, None, db[i:i+c])) + _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -487,6 +513,8 @@ def merge_deltas(dcl, dstreams): # END for each delta stream + # print dcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ diff --git a/stream.py b/stream.py index 3248ae464..0cb558d78 100644 --- a/stream.py +++ b/stream.py @@ -346,6 +346,19 @@ def _set_cache_(self, attr): dc.apply(bbuf, write) # END for each deltachunk to apply + self._mm_target.seek(0) + + ## DEBUG ## + mt = self._mm_target + for ds in self._dstreams: + ds.stream.seek(0) + self._bstream.stream.seek(0) + self._set_cache_old(attr) + + import chardet + if chardet.detect(mt[:])['encoding'] == 'ascii': + assert self._mm_target[:] == mt[:] + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" @@ -399,7 +412,7 @@ def _set_cache_old(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf) + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### # finally, swap out source and target buffers. The target is now the diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx new file mode 100644 index 0000000000000000000000000000000000000000..a7d6c7177ef3ca68b3b93adafa738b190db9e7ac GIT binary patch literal 2248 zcmciEdo&Yz902g;F|X~SRHBG!gxeyIOi6J{XiJoEkwg!9v}mk#)KDpOOOoa!k4*Ad znD;C0xLvwl6=OGfg>ht98CUMtIXTB4?l~RZb9c_??ECxve!uT;e|^834l zlFur*v=R&Nt&}L-{Sjhte~~!ED}jW23nYN#GNd85NCx8XfrfM6>nAw7lq{^5l85zD zt6{z7f2IKM{hM{Lw`e`YixeSV&IZW;C?%+|NEzY<8^75E-~TlgsJTEDY+gbQ;^nAA zegS3~8jxGCWf_`~Tcic?Qd?oYls2rFg^tHg&S9KhKc%fF5m)DGnogOgE9cK~C$WMO z={v>5H?FddyT-|R+8ooZDBBr13G1l(`p)Ymb>YL2_#VbKKEM81Z7lAfb>6yClP)cX z-IsDY8|>}twxKz{$YXc+p$a9R@%Bg9mg5Tp4rL`o3VqnD7g*Ed zx{~AUaaP1Wf~#7IlMkEUflgG5BzXJlX8JYSt7Hf%31j_Amx4J&h32mG<5X|^mz-{v z0B59Ma3tS(ZEeW(Yc-E!!vfb&{P`Nxc%yf*cSCyNO)v6^nzI!_*UE zHlJ`}=uB4W$owt6!*Tv3&qS3bDHnu~FH@nsskcqQ6^Yh)9+XEqU`6y#DxaF#AZomY zb}y}-5-aWau}k?PMsCu_N;ASjne>oFj_GZo;teiRSbcX;+Erd-iiUKOk!4JEa1c4s z!LBr^S8l#V;F>2g;BJ>SniX8xCf#Ug7>;DgZ%!F(eS{i$_wv4a&{e)^jB2xX(bwHY zi5&V?f{?C5NQ?EzRn+3yqhb`_8sHyd8T2N7Ssh||WaC{-qpdi17NhQM^7uf$-~0J$ z2gSUa_3c+BN8gJ5Ws_VKaPso7d>UqT&E$6EnqQ`vt0)Yli(9;s@9m>ET%$f4q^6QY zo@0tLJu1VCqC=a$+#Kq#G(Z39f__z*S3r>t@yYu=X&X=Xbu%qrl(a-w>@VBnQt*d^ z9vMY^FJU+v*+!eJ61g1v)%WKMe;v2nJ#tkAY9j*5n1 zim~2UzDj5TgYDjft#h5;xmG4}2iJygVxpjtQHROSp4b_cQ*Y+>>vb1FM%;j4SF6tu z7FR~L8KoABNT>=c>vCIdshyb|Capo-GR2nm$$kQNJD2W3&>yFDTWH(7c4Tj=HJ!8G zJJCPwUFCJ8WvpYD!HtQYpQm*L(1MB!g5fN>U6+P%?>x&-y#3An^8;APnTL4ijRQuE z$WwdiRFOkHADyPoiJAH^oRaO>Vxig8bk7gI?Cr)DTIXk*lc-7k%4_>eTax^wRjchg z^0wjYKPz$8JQ$9>b?k6-OJ$VmrJG+$)i0FOtwzW}RBHArJC;5D&w6AwZlDG4yHVHRv13FtSg280tXZ zK|8`4da4BUaB&A1dmK1&4(fb>z287M)c6~6(32#Xu>TUkTdV$r_n;@D;dh|nJv1NA x9D{pqvSt~!EoI3lZP9k4v=g3(X<=qg4%Q^Gr|y1tEW5xf@2?fh{cJ?LVl%wKv z^pxJP5=oIZLnI0yh!DSQ+o$<{4#sushM+vou1idydElvH&GMXOwm=VaD??kqO4F_>M_4D*c7}Y%lbdGD7|jlqFbS6b4L_o(2mi^lhrBYBPaBMy20l0zJFs?4ruEBO)n+=jL#_VJqdSJ*20b8ViH{ zwuoTfEIXr<+|m*;6O>E8sC)u#e8JiX>3^v>T18;+nFFhyyy=G{@z)2~*u5*n1C{rz zLiJB?Mn@K=_;jWZ@Y$&d@-4nom^MhLwdiE(Dng#Ba8g{Wf}U#A=gw7cn@eWQl*U;a zA%Ad-&Us{bx93$>Kc})Z8PB_(%a$!|Dng+!lM}12LR|a9D9@u>r*j$j3UwJk#f3rP z?At$`R7-?K2lea!V3~tkm0-X?^!ap>$WT|E$n1$9p!*hYnIAYJUC&PClBwGT!*zMW9Kp(3|Mt9OFsRKKv`Ih0`j{gy98aS<2@FF~Oa>wU zS>YZPeCB{^+gIfnY(H1`z5Ez0<&+f~Njlhu4i?lk&Ql!iJEyNMuo@P7S&d6^wid3o zF*5=eHwqLe0}`P`e4Lk;zi9fHhW48{P8(!5gQTeRZC%gth9B5S*#FD&mu7`_HnFd#N=vNyRVPbT?k?jcZ$ShF6;FjVBt;oo*@jNHr{N&U3Ta4{DqBqd|WWei=TbAWvb_RqhpC5 z$6th{GQK6A2cW+)6g<2_A6-+B#_Qpfd(y_)m``LFyGX|4CQPMOis2qLCh^%`HG?p0 z=(NC%g%K9=u~v+u#+GMk^>?-y<8yw?byu;k^XyPg3J$^kNW`>Meu76zt(Mep%FCiL zXn!Lj?vGbC;C-lpSgV;>SYc_QzQOfkHvH!RxjDfr!jnyX15~VfQnH&F{H{I`GS72L zL{NVjH+I})NuKGYq>Ytel~v%gA_>Zo6|@8+OvrGdIp4aC$X}Sjol)@fBYnWB>&Jtv zh-ga5*&i~oMpORF_b`QYjZoCpzyJesT&kOJGpIPPR_p1n9UkWmrkMsr#bGL~H9UB~ z9hAQQU#cT)W8BKN-Srb(7C3=`Ri3NK%v8>}XlzGR1K|-|CC%{sKQ9AMH^788N~O%2(xw3$ybV#!P%A?&tjj z-|a<&+#jm{eucsYjaWAi$=osckQx3@H?$mgHQ2ceUPPwka`7Jo zjsEu)Di}nAj#wwvYTf#^<6yMG_{0Tq^ADBfe^r*1sPbFwC8nG&9;RmhsjPWSVak#T z5u$HYbzj3^K_mZiEql<6g%6CRZbP!Y7wrSZ>%LcMP~tSN;2QWF-5KcyoCQUxINHlN^&xQTdE!~$W6gR&H9e-KMeP`0wquA@r#MJ$AAYAI8iN*sA?ljAkLtlGW1yE;@t zWQPQ@gfOB6{DOpQ-DM^_JWN_rN9?wkQMY{#1E{LMT>`)(=TsL)R$Vh1jk9`n7fMn{ zdrj)>N?Cnn=1O=~?v!#39CNPT0}2POyzoDP*xiuK*k)K8)tCC$t^Fqbx3e;)y5f@e z`rTpz52?P>R{w2tlRZdy>Ntrzmr4FgsO?z`83cy`!Gh~iyRpR*4usq5cW>dSg&Xk2 zvxy$<7t@wbaWS9HydfP@+nW4%OEr6BgX9&H+TCzarFY4s)o}m1#kb))R)GzEo_6dU z&3j8^drWvetlUi|Kfu!&dQf__hirTQXuLKUqzr@a5Rp-O$Ov_1k(Bb9y>MxVu+ZZ9 z_rleD{DKbpn;Ys~vm3_Kw1EK}FJxFZb5@aa{B6AW!!?tI|86r;UW074U0Pb(>@ zD%-GzhF@C`_uhI%9iyA;o~d)3{5Wv!!LWC_)zFRyFnI$iKw?4@B|2J;#wo#$G{=^V zfdde+6Z#*p@(Dy2$F?Z56^)mj7IRNCZWc<)J)H(#5{TM0Dlb`N#o! z+IxfI3uAJn!^*MdP=h>C*kOXP>0An@7jl_WH81*93j@RFIH)93G%?(+D~O^5(@Anyu2U#}#%!3@3g@`J z{6znRRE5yo1#7NTmHC>oro#hM*G)_-hpM%3Zz9=*lboD}I)XCE)M)H2*AponveGP| zi5d6ORxkyz=*4tWPe8F;T1O}o6{qCkopL_1+Dz-82h8OxALCKtAF?N;^u0ZXcyuxy z!B!uC^wicTVdQd+RWX?(CTy+t6+ZGII^fTZJWZmUo4RIXCac~fge!g9OuKE0GbdIF zydz8E@0h=9S9@rCWP=eX9? zlTwG*y=x;ynHRb=J+FE;pQk^c4`gkN%s->; zSyaOO0IQ^1L`dZ0&Jj@xUmG+_Yn7Sok5_)2>i9b~oxyLOg;w8gTg)x-^md;dSAgNR zxF1cFojbJXB(#C=$+4)azmVi_-}MUDn{uU0DRsA@zk~u$jCO?-UGPgg?=M;|%frgw zLp`aMB7r&MgL0`5WRd3RLFHFVpN>?SiSjL|2nzxEY?RCCHs|M?KOpk)LT9oIS_WNo zTA2NLW4qrsSU-YfXJqs}kwfEl=3duThlD1GT=Zjqc8#!*xP=|O0M2$ln){6^V`&*r zi_;j{HT)+0o1BiHd@?TUJk>1QQ>WMwsD*ET9^PnFR)Z&2-X&IfMgCKfuqx?#!#&8Y z;_MK2OY(%+Xh{&OVJf^ja1KMD&Hkc&pI?b=yGwe%-eT4!9mw=-;K`n>V%Zk}Pv-!JYHwmpvG^M0RM2*)HK7<0FK`<6yhXXOe7pov9#s|RS zL8!y}t=Gt}Lod|sRk;^`p(^~0v3rTdxt;_*3_}uq<9PN5sNo{t7#JBT0GxK9`ojY% zqUv3!pF2LwL9Ir9{+JN2?V;5gu>##-h~L#X$Nd4D1aHL2V*msVS~@J3em+63&olOq z{i=O*Xx#9MLNtr48ADwoyRUCN_W^%m_?s(|d)IS~fp!n;(RCM}$Mnz&*P+9N_J1m1 z^J}M@K0|;G4hX+*{DA>#xq(=I)jb2C1t8=9XF#y=(^;CEbGkV(z+dhn-}3h`voZjv z*#d7IK3tDnYgIZEX|rJY%3(AmV$t{?q;Bw0poR>w@JW*2U*UF=XCV^820)RZ--q?Z zW7qF5)XCtFmMSz;45Us<9w|snM3aVyvXcIMg@=+rv>QWvK@Iru^ zYWBuCP_n@OZR5SKTJG6&DoaSb6jwc$n$k0*sL?jqk>nqCR*diSf}f;>B+#P{K6!XJ zYqsoaUk6 vEUvU}!6G)gfHqQi+dFJwW&vClgS>$17F!H9R$XG(1^|! literal 0 HcmV?d00001 diff --git a/test/test_pack.py b/test/test_pack.py index 717d07e46..770a78bad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -34,8 +34,10 @@ class TestPack(TestBase): packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) def _assert_index_file(self, index, version, size): @@ -123,14 +125,15 @@ def test_pack_index(self): def test_pack(self): # there is this special version 3, but apparently its like 2 ... - for packfile, version, size in (self.packfile_v2_1, self.packfile_v2_2): + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): pack = PackFile(packfile) self._assert_pack_file(pack, version, size) # END for each pack to test def test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2)): + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo indexfile, version, size = indexinfo entity = PackEntity(packfile) From 4cf3eac8e9ed6ac2b0bac4a617a08fdf194da4ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 19:01:24 +0200 Subject: [PATCH 0062/1790] initial frame for design change - now chunks can refer to DeltaLists as well, which is required to get the copying right. This surely makes things more coplex, but should still result in a performance improvement --- fun.py | 91 ++++++++++++++++++++++++++++++++++++++++++++----------- stream.py | 7 ++--- 2 files changed, 75 insertions(+), 23 deletions(-) diff --git a/fun.py b/fun.py index 2d51f62eb..a0835746e 100644 --- a/fun.py +++ b/fun.py @@ -86,13 +86,14 @@ class DeltaChunk(object): 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None - 'data' # chunk of bytes to be added to the target buffer or None + 'data' # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts - self.so = so + self.so = sos self.data = data def __repr__(self): @@ -103,11 +104,15 @@ def __repr__(self): def rbound(self): return self.to + self.ts + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is None or not isinstance(self.data, DeltaChunkList) + def apply(self, source, write): """Apply own data to the target buffer :param source: buffer providing source bytes for copy operations :param write: write method to call with data to write""" - if self.data is None: + if self.has_data(): # COPY DATA FROM SOURCE assert len(source) - self.so - self.ts > 0 write(buffer(source, self.so, self.ts)) @@ -166,7 +171,7 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): def _handle_merge(ld, rd): """Optimize the layout of the lhs delta and the rhs delta TODO: Once the default implementation is working""" - if d.data is None: + if d.has_data(): if od.data: # OVERWRITE DATA pass @@ -217,6 +222,7 @@ def _merge_delta(dcl, dc): _move_delta_lbound(cd, dc.rbound() - cd.to) break else: + # xx.|---| # WE DON'T OVERLAP IT # this can actually happen, once multiple streams are merged break @@ -224,7 +230,7 @@ def _merge_delta(dcl, dc): # END lbound overlap handling else: if dc.to >= cd.rbound(): - #|---|...xx + #|---|xx break # END @@ -269,9 +275,30 @@ def _merge_delta(dcl, dc): class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def terminate_at(self, size): + def init(self, size): + """Intialize this instance with chunks defining to fill up size from a base + buffer of equal size + :return: self""" + if len(self) != 0: + return + # pretend we have one huge delta chunk, which just copies everything + # from source to destination + maxint32 = 2**32 + for x in range(0, size, maxint32): + self.append(DeltaChunk(x, maxint32, x, None)) + # END create copy chunks + offset = x*maxint32 + remainder = size-offset + if remainder: + self.append(DeltaChunk(offset, remainder, offset, None)) + # END handle all done in loop + + return self + + def set_rbound(self, size): """Chops the list at the given size, splitting and removing DeltaNodes - as required""" + as required + :return: self""" di = _closest_index(self, size) d = self[di] rsize = size - d.to @@ -283,6 +310,26 @@ def terminate_at(self, size): ## DEBUG ## self.check_integrity(size) + return self + + def connect_with(self, bdlc): + """Connect this instance's delta chunks virtually with the given base. + This means that all copy deltas will simply apply to the given region + of the given base. Afterwards, the base is optimized so that add-deltas + will be truncated to the region actually used, or removed completely where + adequate. This way, memory usage is reduced. + :param bdlc: DeltaChunkList to serve as base""" + raise NotImplementedError("todo") + + def apply(self, bbuf, write): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + raise NotImplementedError("todo") + def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -437,31 +484,31 @@ def stream_copy(read, write, size, chunk_size): def reverse_merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: see merge_deltas + :param dcl: see 3 :param dstreams: iterable of delta stream objects. They must be ordered latest first, hence the delta to be applied last comes first, then its ancestors :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def merge_deltas(dcl, dstreams): +def merge_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: DeltaChunkList, may be empty initially, and will be changed - during the merge process :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first - :return: None""" + :return: DeltaChunkList, containing all operations to apply""" + bdcl = None # data chunk list for initial base + dcl = DeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() delta_buf_size = ds.size # read header - i, src_size = msb_size(db) + i, base_size = msb_size(db) i, target_size = msb_size(db, i) # interpret opcodes - tbw = 0 # amount of target bytes written + tbw = 0 # amount of target bytes written while i < delta_buf_size: c = ord(db[i]) i += 1 @@ -494,14 +541,16 @@ def merge_deltas(dcl, dstreams): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_size): + rbound > base_size): break - _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + # _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) + # _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -509,8 +558,14 @@ def merge_deltas(dcl, dstreams): # END handle command byte # END while processing delta data - dcl.terminate_at(target_size) + # merge the lists ! + if base is not None: + dcl.connect_with(base) + # END handle merge + # prepare next base + base = dcl + dcl = DeltaChunkList() # END for each delta stream # print dcl diff --git a/stream.py b/stream.py index 0cb558d78..efb99d218 100644 --- a/stream.py +++ b/stream.py @@ -325,8 +325,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = DeltaChunkList() - merge_deltas(dcl, reversed(self._dstreams)) + dcl = merge_deltas(reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -342,9 +341,7 @@ def _set_cache_(self, attr): # APPLY CHUNKS write = self._mm_target.write - for dc in dcl: - dc.apply(bbuf, write) - # END for each deltachunk to apply + dcl.apply(bbuf, write) self._mm_target.seek(0) From fbf8f3221ea58de54567e034d8a7b4e23833e14b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 12:35:35 +0200 Subject: [PATCH 0063/1790] Filled in first implementation of all missing methods. Its untested, and currently the lists are truncated physically, which would work, but it would be easier to just remember the changed bounds, and apply it later with these bounds in mind --- fun.py | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 168 insertions(+), 22 deletions(-) diff --git a/fun.py b/fun.py index a0835746e..3e126c3e4 100644 --- a/fun.py +++ b/fun.py @@ -12,6 +12,8 @@ import mmap from itertools import islice, izip +from copy import copy + # INVARIANTS OFS_DELTA = 6 REF_DELTA = 7 @@ -51,33 +53,48 @@ def _set_delta_rbound(d, size): """Truncate the given delta to the given size :param size: size relative to our target offset, may not be 0, must be smaller or equal - to our size""" + to our size + :return: d""" if size == 0: raise ValueError("size to truncate to must not be 0") if d.ts == size: return if size > d.ts: - raise ValueError("Cannot truncate delta 'larger'") + raise ValueError("Cannot extend rbound") d.ts = size # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE, see _split_delta + + if d.has_copy_chunklist(): + d.data.set_rbound(size) + # END truncate chunklist + + return d def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static - :param bytes: amount of bytes to move, must be smaller than delta size""" + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return if bytes >= d.ts: raise ValueError("Cannot move offset that much") d.to += bytes d.so += bytes d.ts -= bytes - if d.data: - d.data = d.data[bytes:] + if d.data is not None: + if isinstance(d.data, DeltaChunkList): + d.data.move_lbound(bytes) + else: + d.data = d.data[bytes:] + # END handle data type # END handle data + return d class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -106,16 +123,22 @@ def rbound(self): def has_data(self): """:return: True if the instance has data to add to the target stream""" - return self.data is None or not isinstance(self.data, DeltaChunkList) + return self.data is not None and not isinstance(self.data, DeltaChunkList) - def apply(self, source, write): + def has_copy_chunklist(self): + """:return: True if we copy our data from a chunklist""" + return return self.data is not None and isinstance(self.data, DeltaChunkList) + + def apply(self, bbuf, write): """Apply own data to the target buffer - :param source: buffer providing source bytes for copy operations + :param bbuf: buffer providing source bytes for copy operations :param write: write method to call with data to write""" - if self.has_data(): + if self.data is None: # COPY DATA FROM SOURCE - assert len(source) - self.so - self.ts > 0 - write(buffer(source, self.so, self.ts)) + assert len(bbuf) - self.so - self.ts > 0 + write(buffer(bbuf, self.so, self.ts)) + elif isinstance(self.data, DeltaChunkList): + self.data.apply(bbuf, write) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -153,6 +176,8 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): :note: belongs to DeltaChunkList""" if relofs > d.ts: raise ValueError("Cannot split behinds a chunks rbound") + if relofs < 1: + raise ValueError("Cannot split delta with %i" % relofs) osize = d.ts - relofs _set_delta_rbound(d, relofs) @@ -295,23 +320,65 @@ def init(self, size): return self - def set_rbound(self, size): - """Chops the list at the given size, splitting and removing DeltaNodes + def set_rbound(self, relofs): + """Chops the list at the given relative offset, splitting and removing DeltaNodes as required + :param relofs: offset relative to the start of the chain :return: self""" - di = _closest_index(self, size) + if len(self) == 0: + raise AssertionError("Cannot change bound of empty list") + if relofs == 0: + raise ValueError("Size to truncate to must not be 0") + absofs = self.lbound() + relofs + if absofs > self.rbound(): + raise ValueError("Cannot extend chunk list") + di = _closest_index(self, absofs) d = self[di] - rsize = size - d.to + rsize = absofs - d.to if rsize: _set_delta_rbound(d, rsize) # END truncate last node if possible del(self[di+(rsize!=0):]) ## DEBUG ## - self.check_integrity(size) + self.check_integrity(absofs) return self + def move_lbound(self, bytes): + """Offset the left bound of the list by the given amount of bytes. + This effectively truncates the list + :return: self""" + if len(self) == 0: + raise AssertionError("Cannot change bound of empty list") + if bytes == 0: + return + abslbound = self.lbound() + bytes + if abslbound >= self.rbound(): + raise ValueError("Cannot move lbound that much") + + dsi = _closest_index(self, abslbound) + d = self[dsi] + _move_delta_lbound(d, abslbound - d.to) + + if dsi: + del(self[:dsi]) + # END remove all skipped nodes + + return self + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + def connect_with(self, bdlc): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region @@ -319,7 +386,11 @@ def connect_with(self, bdlc): will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. :param bdlc: DeltaChunkList to serve as base""" - raise NotImplementedError("todo") + for dc in self: + if not dc.has_data(): + dc.data = bdcl[dc.to, dc.ts] + # END handle overlap + # END for each dc def apply(self, bbuf, write): """Apply the chain's changes and write the final result using the passed @@ -328,7 +399,10 @@ def apply(self, bbuf, write): list. It will only be used if the chunk in question does not have a base chain. :param write: function taking a string of bytes to write to the output""" - raise NotImplementedError("todo") + dapply = DeltaChunk.apply + for dc in self: + dapply(dc, bbuf, write) + # END for each dc def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches @@ -345,6 +419,7 @@ def check_integrity(self, target_size=-1): # check data for dc in self: + assert dc.ts > 0 if dc.data: assert len(dc.data) >= dc.ts # END for each dc @@ -359,6 +434,77 @@ def check_integrity(self, target_size=-1): assert lft.to + lft.ts == rgt.to # END for each pair + def __getslice__(self, absofs, size): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: DeltaChunkList (copy) which represents the given chunk""" + cdi = _closest_index(self, absofs) # delta start index + slen = len(self) + ndcl = self.__class__() + rbound = absofs + size + + while cdi < slen: + # are we larger than the current block + cd = self[cdi] + if absofs < cd.to: + if rbound >= cd.rbound(): + # xxx|xxx|x + # cd is fully contained in the range + ndcl.append(copy(cd)) + elif rbound > cd.to: + # partially contained + # xxx|x--| + cd = copy(cd) + _set_delta_rbound(cd, cd.rbound() - rbound) + ndcl.append(cd) + break + else: + # xx.|---| + # WE DON'T OVERLAP IT + break + # END rbound overlap handling + # END lbound overlap handling + else: + if absofs >= cd.rbound(): + # happens if slice is out of bound + #|---|xx + break + # END + + if rbound >= cd.rbound(): + if absofs == cd.to: + #|xxx|x + # fully contained + ndcl.append(copy(cd)) + else: + # shift + #|-xx| + cd = copy(cd) + _move_delta_lbound(cd, absofs - cd.to) + ndcl.append(cd) + # END handle offset special case + elif absofs == cd.to: + #|x--| + # we truncate it to our size + cd = copy(cd) + _set_delta_rbound(cd, size) + ndcl.append(cd) + break + else: + #|-x-| + # adjust both ends + cd = copy(cd) + _move_delta_lbound(cd, absofs - cd.to) + _set_delta_rbound(cd, size) + ndcl.append(cd) + break + # END handle rbound overlap + # END handle overlap + # END for each chunk + return ndcl + + + #} END structures #{ Routines @@ -559,16 +705,16 @@ def merge_deltas(dstreams): # END while processing delta data # merge the lists ! - if base is not None: - dcl.connect_with(base) + if bdcl is not None: + dcl.connect_with(bdcl) # END handle merge # prepare next base - base = dcl + bdcl = dcl dcl = DeltaChunkList() # END for each delta stream - # print dcl + return base def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): From 1c2caf590d85866d95c3d1470eba692a61de3622 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 19:37:22 +0200 Subject: [PATCH 0064/1790] Forward Delta Application now appears to work --- fun.py | 419 +++++++++++++++++------------------------------------- stream.py | 18 ++- 2 files changed, 146 insertions(+), 291 deletions(-) diff --git a/fun.py b/fun.py index 3e126c3e4..ac0b1c098 100644 --- a/fun.py +++ b/fun.py @@ -44,8 +44,8 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'reverse_merge_deltas', - 'merge_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'reverse_connect_deltas', + 'connect_deltas', 'DeltaChunkList') #{ Structures @@ -55,21 +55,17 @@ def _set_delta_rbound(d, size): :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size :return: d""" - if size == 0: - raise ValueError("size to truncate to must not be 0") if d.ts == size: return + if size == 0: + raise ValueError("size to truncate to must not be 0") if size > d.ts: raise ValueError("Cannot extend rbound") d.ts = size # NOTE: data is truncated automatically when applying the delta - # MUST NOT DO THIS HERE, see _split_delta - - if d.has_copy_chunklist(): - d.data.set_rbound(size) - # END truncate chunklist + # MUST NOT DO THIS HERE return d @@ -85,13 +81,10 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes + d.sob += bytes d.ts -= bytes - if d.data is not None: - if isinstance(d.data, DeltaChunkList): - d.data.move_lbound(bytes) - else: - d.data = d.data[bytes:] - # END handle data type + if d.has_data(): + d.data = d.data[bytes:] # END handle data return d @@ -103,14 +96,16 @@ class DeltaChunk(object): 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None - 'data' # chunk of bytes to be added to the target buffer, + 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None + 'sob' # DEBUG: Backup ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts - self.so = sos + self.so = so + self.sob = so self.data = data def __repr__(self): @@ -118,6 +113,18 @@ def __repr__(self): #{ Interface + def copy_offset(self): + """:return: offset to apply when copying from a base buffer, or 0 + if this is not a copying delta chunk""" + + if self.data is not None: + if isinstance(self.data, DeltaChunkList): + return self.data.lbound() + self.so + else: + return self.so + # END handle data type + return 0 + def rbound(self): return self.to + self.ts @@ -127,7 +134,15 @@ def has_data(self): def has_copy_chunklist(self): """:return: True if we copy our data from a chunklist""" - return return self.data is not None and isinstance(self.data, DeltaChunkList) + return self.data is not None and isinstance(self.data, DeltaChunkList) + + def set_copy_chunklist(self, dcl): + """Set the deltachunk list to be used as basis for copying. + :note: only works if this chunk is a copy delta chunk""" + assert self.data is None, "Cannot assign chain to add delta chunk" + self.data = dcl + self.sob = self.so + self.so = 0 # allows lbound moves to be virtual def apply(self, bbuf, write): """Apply own data to the target buffer @@ -135,10 +150,10 @@ def apply(self, bbuf, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE - assert len(bbuf) - self.so - self.ts > 0 + assert len(bbuf) - self.so - self.ts > -1 write(buffer(bbuf, self.so, self.ts)) elif isinstance(self.data, DeltaChunkList): - self.data.apply(bbuf, write) + self.data.apply(bbuf, write, self.so, self.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -165,208 +180,10 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 -def _split_delta(dcl, d, di, relofs, insert_offset=0): - """Split the delta at di into two deltas, adjusting their sizes, offsets and data - accordingly and adding the new part to the dcl - :param relofs: relative offset at which to split the delta - :param d: delta chunk to split - :param di: index of d in dcl - :param insert_offset: offset for the new split id - :return: newly created DeltaChunk - :note: belongs to DeltaChunkList""" - if relofs > d.ts: - raise ValueError("Cannot split behinds a chunks rbound") - if relofs < 1: - raise ValueError("Cannot split delta with %i" % relofs) - - osize = d.ts - relofs - _set_delta_rbound(d, relofs) - - # insert new one - drb = d.rbound() - - nd = DeltaChunk( drb, - osize, - d.so + osize, - (d.data and d.data[osize:]) or None ) - - self.insert(di+1+insert_offset, nd) - return nd - -def _handle_merge(ld, rd): - """Optimize the layout of the lhs delta and the rhs delta - TODO: Once the default implementation is working""" - if d.has_data(): - if od.data: - # OVERWRITE DATA - pass - else: - # MERGE SOURCE AREA - pass - # END overwrite data - else: - if od.data: - # MERGE DATA WITH DATA - # overwrite the data at the respective spot - pass - else: - # INSERT DATA INTO COPY AREA - pass - # END combine or insert data - # END handle chunk mode - -def _merge_delta(dcl, dc): - """Merge the given DeltaChunk instance into the dcl - :param d: the DeltaChunk to merge""" - if len(dcl) == 0: - dcl.append(dc) - return - # END early return on empty list - - cdi = _closest_index(dcl, dc.to) # current delta index - cd = dcl[cdi] # current delta - - # either we go at his spot, or after - # cdi either moves one up, or stays - #print "insert at %i" % (cdi + (dc.to > cd.to)) - #print cd, dc - dcl.insert(cdi + (dc.to > cd.to), dc) - cdi += dc.to == cd.to - - while True: - # are we larger than the current block - if dc.to < cd.to: - if dc.rbound() >= cd.rbound(): - # xxx|xxx|x - # remove the current item completely - dcl.pop(cdi) - cdi -= 1 - elif dc.rbound() > cd.to: - # MOVE ITS LBOUND - # xxx|x--| - _move_delta_lbound(cd, dc.rbound() - cd.to) - break - else: - # xx.|---| - # WE DON'T OVERLAP IT - # this can actually happen, once multiple streams are merged - break - # END rbound overlap handling - # END lbound overlap handling - else: - if dc.to >= cd.rbound(): - #|---|xx - break - # END - - if dc.rbound() >= cd.rbound(): - if dc.to == cd.to: - #|xxx|x - # REMOVE CD - dcl.pop(cdi) - cdi -= 1 - else: - # TRUNCATE CD - #|-xx| - _set_delta_rbound(cd, dc.to - cd.to) - # END handle offset special case - elif dc.to == cd.to: - #|x--| - # we shift it by our size - _move_delta_lbound(cd, dc.ts) - else: - #|-x-| - # SPLIT CD AND LBOUND MOVE ITS SECOND PART - # insert offset is required to insert it after us - nd = _split_delta(dcl, cd, cdi, 1) - _move_delta_lbound(nd, dc.ts) - break - # END handle rbound overlap - # END handle overlap - - cdi += 1 - if cdi < len(dcl): - cd = dcl[cdi] - else: - break - # END check for end of list - # while our chunk is not completely done - - ## DEBUG ## - dcl.check_integrity() - - class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def init(self, size): - """Intialize this instance with chunks defining to fill up size from a base - buffer of equal size - :return: self""" - if len(self) != 0: - return - # pretend we have one huge delta chunk, which just copies everything - # from source to destination - maxint32 = 2**32 - for x in range(0, size, maxint32): - self.append(DeltaChunk(x, maxint32, x, None)) - # END create copy chunks - offset = x*maxint32 - remainder = size-offset - if remainder: - self.append(DeltaChunk(offset, remainder, offset, None)) - # END handle all done in loop - - return self - - def set_rbound(self, relofs): - """Chops the list at the given relative offset, splitting and removing DeltaNodes - as required - :param relofs: offset relative to the start of the chain - :return: self""" - if len(self) == 0: - raise AssertionError("Cannot change bound of empty list") - if relofs == 0: - raise ValueError("Size to truncate to must not be 0") - absofs = self.lbound() + relofs - if absofs > self.rbound(): - raise ValueError("Cannot extend chunk list") - di = _closest_index(self, absofs) - d = self[di] - rsize = absofs - d.to - if rsize: - _set_delta_rbound(d, rsize) - # END truncate last node if possible - del(self[di+(rsize!=0):]) - - ## DEBUG ## - self.check_integrity(absofs) - - return self - - def move_lbound(self, bytes): - """Offset the left bound of the list by the given amount of bytes. - This effectively truncates the list - :return: self""" - if len(self) == 0: - raise AssertionError("Cannot change bound of empty list") - if bytes == 0: - return - abslbound = self.lbound() + bytes - if abslbound >= self.rbound(): - raise ValueError("Cannot move lbound that much") - - dsi = _closest_index(self, abslbound) - d = self[dsi] - _move_delta_lbound(d, abslbound - d.to) - - if dsi: - del(self[:dsi]) - # END remove all skipped nodes - - return self - def rbound(self): """:return: rightmost extend in bytes, absolute""" if len(self) == 0: @@ -379,30 +196,84 @@ def lbound(self): return 0 return self[0].to - def connect_with(self, bdlc): + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def connect_with(self, bdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdlc: DeltaChunkList to serve as base""" + :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - dc.data = bdcl[dc.to, dc.ts] + # dc.set_copy_chunklist(bdcl[dc.copy_offset():dc.ts]) + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) # END handle overlap # END for each dc - def apply(self, bbuf, write): + def apply(self, bbuf, write, lbound_offset=0, size=0): """Apply the chain's changes and write the final result using the passed write function. :param bbuf: base buffer containing the base of all deltas contained in this list. It will only be used if the chunk in question does not have a base chain. + :param lbound_offset: offset at which to start applying the delta, relative to + our lbound + :param size: if larger than 0, only the given amount of bytes will be applied :param write: function taking a string of bytes to write to the output""" + slen = len(self) + if slen == 0: + return + # END early abort + absofs = self.lbound() + lbound_offset + if size == 0: + size = self.rbound() - absofs + # END initialize size + if absofs + size > self.rbound(): + raise ValueError("Cannot apply more bytes than there are in this chain") + # END sanity check + + if size > self.rbound() - absofs: + raise ValueError("Trying to apply more than there is available") + dapply = DeltaChunk.apply - for dc in self: - dapply(dc, bbuf, write) - # END for each dc + if lbound_offset or absofs + size != self.rbound(): + cdi = _closest_index(self, absofs) + cd = self[cdi] + if cd.to != absofs: + tcd = copy(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + dapply(tcd, bbuf, write) + size -= tcd.ts + cdi += 1 + # END handle first chunk + + # here we have to either apply full chunks, or smaller ones, but + # we always start at the chunks target offset + while cdi < slen and size: + cd = self[cdi] + if cd.ts <= size: + dapply(cd, bbuf, write) + size -= cd.ts + else: + tcd = copy(cd) + _set_delta_rbound(tcd, size) + dapply(tcd, bbuf, write) + size -= tcd.ts + break + # END handle bytes to apply + cdi += 1 + # END handle rest + assert size == 0 + else: + for dc in self: + dapply(dc, bbuf, write) + # END for each dc + # END handle application values def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches @@ -420,8 +291,10 @@ def check_integrity(self, target_size=-1): # check data for dc in self: assert dc.ts > 0 - if dc.data: + if dc.has_data(): assert len(dc.data) >= dc.ts + if dc.has_copy_chunklist(): + assert dc.ts <= dc.data.size() # END for each dc left = islice(self, 0, len(self)-1) @@ -438,69 +311,43 @@ def __getslice__(self, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: DeltaChunkList (copy) which represents the given chunk""" + if len(self) == 0: + return DeltaChunkList() + + absofs = max(absofs, self.lbound()) + size = min(self.rbound() - self.lbound(), size) cdi = _closest_index(self, absofs) # delta start index + cd = self[cdi] slen = len(self) ndcl = self.__class__() - rbound = absofs + size - while cdi < slen: + if cd.to != absofs: + tcd = copy(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + ndcl.append(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: # are we larger than the current block cd = self[cdi] - if absofs < cd.to: - if rbound >= cd.rbound(): - # xxx|xxx|x - # cd is fully contained in the range - ndcl.append(copy(cd)) - elif rbound > cd.to: - # partially contained - # xxx|x--| - cd = copy(cd) - _set_delta_rbound(cd, cd.rbound() - rbound) - ndcl.append(cd) - break - else: - # xx.|---| - # WE DON'T OVERLAP IT - break - # END rbound overlap handling - # END lbound overlap handling + if cd.ts <= size: + ndcl.append(copy(cd)) + size -= cd.ts else: - if absofs >= cd.rbound(): - # happens if slice is out of bound - #|---|xx - break - # END - - if rbound >= cd.rbound(): - if absofs == cd.to: - #|xxx|x - # fully contained - ndcl.append(copy(cd)) - else: - # shift - #|-xx| - cd = copy(cd) - _move_delta_lbound(cd, absofs - cd.to) - ndcl.append(cd) - # END handle offset special case - elif absofs == cd.to: - #|x--| - # we truncate it to our size - cd = copy(cd) - _set_delta_rbound(cd, size) - ndcl.append(cd) - break - else: - #|-x-| - # adjust both ends - cd = copy(cd) - _move_delta_lbound(cd, absofs - cd.to) - _set_delta_rbound(cd, size) - ndcl.append(cd) - break - # END handle rbound overlap - # END handle overlap + tcd = copy(cd) + _set_delta_rbound(tcd, size) + ndcl.append(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 # END for each chunk + assert size == 0, "size was %i" % size + + ndcl.check_integrity() return ndcl @@ -627,7 +474,7 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw -def reverse_merge_deltas(dcl, dstreams): +def reverse_connect_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dcl: see 3 @@ -636,7 +483,7 @@ def reverse_merge_deltas(dcl, dstreams): :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def merge_deltas(dstreams): +def connect_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dstreams: iterable of delta stream objects. They must be ordered latest last, @@ -690,12 +537,10 @@ def merge_deltas(dstreams): rbound > base_size): break - # _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - # _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c @@ -709,12 +554,14 @@ def merge_deltas(dstreams): dcl.connect_with(bdcl) # END handle merge + dcl.check_integrity() + # prepare next base bdcl = dcl dcl = DeltaChunkList() # END for each delta stream - return base + return bdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): diff --git a/stream.py b/stream.py index efb99d218..8b8655e86 100644 --- a/stream.py +++ b/stream.py @@ -8,7 +8,7 @@ msb_size, stream_copy, apply_delta_data, - merge_deltas, + connect_deltas, DeltaChunkList, delta_types ) @@ -325,7 +325,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = merge_deltas(reversed(self._dstreams)) + dcl = connect_deltas(reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -333,7 +333,7 @@ def _set_cache_(self, attr): return # END handle empty list - self._size = dcl[-1].rbound() + self._size = dcl.rbound() self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) @@ -353,8 +353,16 @@ def _set_cache_(self, attr): self._set_cache_old(attr) import chardet - if chardet.detect(mt[:])['encoding'] == 'ascii': - assert self._mm_target[:] == mt[:] + + print "num dstreams", len(self._dstreams) + #if chardet.detect(mt[:self._size])['encoding'] == 'ascii': + if self._mm_target[:self._size] != mt[:]: + open("working.txt", "w").write(self._mm_target[:self._size]) + open("incorrect.txt", "w").write(mt[:]) + raise AssertionError("Output didn't match") + # END debug + print "success" + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" From 834f081232cb51c251e8f2d3931c9e1fd5eff457 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:07:28 +0200 Subject: [PATCH 0065/1790] Implemented add-chunk compression, which clearly reduces chain size, but might not really be worth it in python --- fun.py | 70 +++++++++++++++++++++++++++++++++++++++++-------------- stream.py | 29 +++++++---------------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/fun.py b/fun.py index ac0b1c098..170b3db15 100644 --- a/fun.py +++ b/fun.py @@ -13,6 +13,7 @@ from itertools import islice, izip from copy import copy +from cStringIO import StringIO # INVARIANTS OFS_DELTA = 6 @@ -57,10 +58,6 @@ def _set_delta_rbound(d, size): :return: d""" if d.ts == size: return - if size == 0: - raise ValueError("size to truncate to must not be 0") - if size > d.ts: - raise ValueError("Cannot extend rbound") d.ts = size @@ -76,8 +73,6 @@ def _move_delta_lbound(d, bytes): :return: d""" if bytes == 0: return - if bytes >= d.ts: - raise ValueError("Cannot move offset that much") d.to += bytes d.so += bytes @@ -139,7 +134,6 @@ def has_copy_chunklist(self): def set_copy_chunklist(self, dcl): """Set the deltachunk list to be used as basis for copying. :note: only works if this chunk is a copy delta chunk""" - assert self.data is None, "Cannot assign chain to add delta chunk" self.data = dcl self.sob = self.so self.so = 0 # allows lbound moves to be virtual @@ -150,13 +144,13 @@ def apply(self, bbuf, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE - assert len(bbuf) - self.so - self.ts > -1 write(buffer(bbuf, self.so, self.ts)) elif isinstance(self.data, DeltaChunkList): self.data.apply(bbuf, write, self.so, self.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it if self.ts < len(self.data): write(self.data[:self.ts]) else: @@ -209,11 +203,54 @@ def connect_with(self, bdcl): :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - # dc.set_copy_chunklist(bdcl[dc.copy_offset():dc.ts]) dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) # END handle overlap # END for each dc + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + slen_orig = slen + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if not dc.has_data(): + if first_data_index is not None and i-2-first_data_index > 1: + #if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in xrange(first_data_index, i-1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i-1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i-1 + # END iterate list + + #if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + def apply(self, bbuf, write, lbound_offset=0, size=0): """Apply the chain's changes and write the final result using the passed write function. @@ -232,12 +269,6 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): if size == 0: size = self.rbound() - absofs # END initialize size - if absofs + size > self.rbound(): - raise ValueError("Cannot apply more bytes than there are in this chain") - # END sanity check - - if size > self.rbound() - absofs: - raise ValueError("Trying to apply more than there is available") dapply = DeltaChunk.apply if lbound_offset or absofs + size != self.rbound(): @@ -347,7 +378,7 @@ def __getslice__(self, absofs, size): # END for each chunk assert size == 0, "size was %i" % size - ndcl.check_integrity() + # ndcl.check_integrity() return ndcl @@ -540,7 +571,8 @@ def connect_deltas(dstreams): dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: - # TODO: Concatenate multiple deltachunks + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c @@ -549,12 +581,14 @@ def connect_deltas(dstreams): # END handle command byte # END while processing delta data + dcl.compress() + # merge the lists ! if bdcl is not None: dcl.connect_with(bdcl) # END handle merge - dcl.check_integrity() + # dcl.check_integrity() # prepare next base bdcl = dcl diff --git a/stream.py b/stream.py index 8b8655e86..098d27a2b 100644 --- a/stream.py +++ b/stream.py @@ -322,6 +322,14 @@ def __init__(self, stream_list): self._br = 0 def _set_cache_(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python + #print "num streams", len(self._dstreams) + #if len(self._dstreams) == 1 or (len(self._dstreams) * self._dstreams.size) > 25*1000*1000: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. @@ -345,26 +353,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - ## DEBUG ## - mt = self._mm_target - for ds in self._dstreams: - ds.stream.seek(0) - self._bstream.stream.seek(0) - self._set_cache_old(attr) - - import chardet - - print "num dstreams", len(self._dstreams) - #if chardet.detect(mt[:self._size])['encoding'] == 'ascii': - if self._mm_target[:self._size] != mt[:]: - open("working.txt", "w").write(self._mm_target[:self._size]) - open("incorrect.txt", "w").write(mt[:]) - raise AssertionError("Output didn't match") - # END debug - print "success" - - - def _set_cache_old(self, attr): + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From bda5ef5e94161c304d6151785b34a20bdb306389 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:32:01 +0200 Subject: [PATCH 0066/1790] implemented binary tree search to get the closest deltachunk by offset --- fun.py | 18 ++++++++++++------ test/test_pack.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index 170b3db15..8f38fa467 100644 --- a/fun.py +++ b/fun.py @@ -165,12 +165,18 @@ def _closest_index(dcl, absofs): to the DeltaChunk with a target buffer absofs that equals or is greater than absofs. :note: global method for performance only, it belongs to DeltaChunkList""" - # TODO: binary search !! - for i,d in enumerate(dcl): - if absofs < d.to: - return i-1 - elif absofs == d.to: - return i + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound # END for each delta absofs return len(dcl)-1 diff --git a/test/test_pack.py b/test/test_pack.py index 770a78bad..6e598d755 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def test_pack_entity(self): + def _test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): From 5d18685948602de69eb950d0238fcf80f0413b68 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:53:11 +0200 Subject: [PATCH 0067/1790] Disabled delta-aggregation as it is reduces the throughput to 540KiB/s compared to 9.4MiB compared to the previous brute-force algorithm. Compression helps, but it would probably be more efficient if done right away, not as post-process. It might help to implement the reversed version of this algorithm, as initially intended, but currently the overhead is the actual application --- fun.py | 4 ---- stream.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index 8f38fa467..bc67e0fce 100644 --- a/fun.py +++ b/fun.py @@ -76,7 +76,6 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes - d.sob += bytes d.ts -= bytes if d.has_data(): d.data = d.data[bytes:] @@ -93,14 +92,12 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - 'sob' # DEBUG: Backup ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts self.so = so - self.sob = so self.data = data def __repr__(self): @@ -135,7 +132,6 @@ def set_copy_chunklist(self, dcl): """Set the deltachunk list to be used as basis for copying. :note: only works if this chunk is a copy delta chunk""" self.data = dcl - self.sob = self.so self.so = 0 # allows lbound moves to be virtual def apply(self, bbuf, write): diff --git a/stream.py b/stream.py index 098d27a2b..7347f527c 100644 --- a/stream.py +++ b/stream.py @@ -311,6 +311,10 @@ class DeltaApplyReader(LazyMixin): "_br" # number of bytes read ) + #{ Configuration + k_max_memory_move = 250*1000*1000 + #} END configuration + def __init__(self, stream_list): """Initialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the @@ -325,8 +329,9 @@ def _set_cache_(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python - #print "num streams", len(self._dstreams) - #if len(self._dstreams) == 1 or (len(self._dstreams) * self._dstreams.size) > 25*1000*1000: + # hence we apply a worst-case scenario here + # TODO: read the final size from the deltastream - have to partly unpack + # if len(self._dstreams) * self._size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) @@ -353,7 +358,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_brute_(self, attr): + def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From 682f483fa61c77fd6121ae860002094eb517bbd4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 00:07:37 +0200 Subject: [PATCH 0068/1790] First profiling run revealed that the copy function was a serious slowdown. Now its twice as fast compared to the previous version, but still about 8 times slower than the brute force approach --- fun.py | 14 ++++++++------ stream.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index bc67e0fce..9f2c9b6be 100644 --- a/fun.py +++ b/fun.py @@ -12,7 +12,6 @@ import mmap from itertools import islice, izip -from copy import copy from cStringIO import StringIO # INVARIANTS @@ -82,6 +81,9 @@ def _move_delta_lbound(d, bytes): # END handle data return d + +def delta_duplicate(src): + return DeltaChunk(src.to, src.ts, src.so, src.data) class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -277,7 +279,7 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): cdi = _closest_index(self, absofs) cd = self[cdi] if cd.to != absofs: - tcd = copy(cd) + tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) dapply(tcd, bbuf, write) @@ -293,7 +295,7 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): dapply(cd, bbuf, write) size -= cd.ts else: - tcd = copy(cd) + tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) dapply(tcd, bbuf, write) size -= tcd.ts @@ -355,7 +357,7 @@ def __getslice__(self, absofs, size): ndcl = self.__class__() if cd.to != absofs: - tcd = copy(cd) + tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) ndcl.append(tcd) @@ -367,10 +369,10 @@ def __getslice__(self, absofs, size): # are we larger than the current block cd = self[cdi] if cd.ts <= size: - ndcl.append(copy(cd)) + ndcl.append(delta_duplicate(cd)) size -= cd.ts else: - tcd = copy(cd) + tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) ndcl.append(tcd) size -= tcd.ts diff --git a/stream.py b/stream.py index 7347f527c..16c917a97 100644 --- a/stream.py +++ b/stream.py @@ -358,7 +358,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_(self, attr): + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From a064007f0e25b8a4af04c30fe329e345edc8092e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 08:08:09 +0200 Subject: [PATCH 0069/1790] Made heavliy called methods global, its brings a second, which is nearly 10 percent more performance just by eliminating two method calls --- fun.py | 154 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 79 insertions(+), 75 deletions(-) diff --git a/fun.py b/fun.py index 9f2c9b6be..253e64092 100644 --- a/fun.py +++ b/fun.py @@ -84,6 +84,26 @@ def _move_delta_lbound(d, bytes): def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) + +def delta_chunk_apply(dc, bbuf, write): + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(buffer(bbuf, dc.so, dc.ts)) + elif isinstance(dc.data, DeltaChunkList): + delta_list_apply(dc.data, bbuf, write, dc.so, dc.ts) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -136,25 +156,7 @@ def set_copy_chunklist(self, dcl): self.data = dcl self.so = 0 # allows lbound moves to be virtual - def apply(self, bbuf, write): - """Apply own data to the target buffer - :param bbuf: buffer providing source bytes for copy operations - :param write: write method to call with data to write""" - if self.data is None: - # COPY DATA FROM SOURCE - write(buffer(bbuf, self.so, self.ts)) - elif isinstance(self.data, DeltaChunkList): - self.data.apply(bbuf, write, self.so, self.ts) - else: - # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? - # Considering data can be larger than 127 bytes now, it should be worth it - if self.ts < len(self.data): - write(self.data[:self.ts]) - else: - write(self.data) - # END handle truncation - # END handle chunk mode + #} END interface @@ -178,6 +180,59 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 +def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param lbound_offset: offset at which to start applying the delta, relative to + our lbound + :param size: if larger than 0, only the given amount of bytes will be applied + :param write: function taking a string of bytes to write to the output""" + slen = len(dcl) + if slen == 0: + return + # END early abort + absofs = dcl.lbound() + lbound_offset + if size == 0: + size = dcl.rbound() - absofs + # END initialize size + + if lbound_offset or absofs + size != dcl.rbound(): + cdi = _closest_index(dcl, absofs) + cd = dcl[cdi] + if cd.to != absofs: + tcd = delta_duplicate(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + delta_chunk_apply(tcd, bbuf, write) + size -= tcd.ts + cdi += 1 + # END handle first chunk + + # here we have to either apply full chunks, or smaller ones, but + # we always start at the chunks target offset + while cdi < slen and size: + cd = dcl[cdi] + if cd.ts <= size: + delta_chunk_apply(cd, bbuf, write) + size -= cd.ts + else: + tcd = delta_duplicate(cd) + _set_delta_rbound(tcd, size) + delta_chunk_apply(tcd, bbuf, write) + size -= tcd.ts + break + # END handle bytes to apply + cdi += 1 + # END handle rest + else: + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc + # END handle application values + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" @@ -211,6 +266,11 @@ def connect_with(self, bdcl): # END handle overlap # END for each dc + def apply(self, bbuf, write, lbound_offset=0, size=0): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write, lbound_offset, size) + def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate add-chunks @@ -255,61 +315,6 @@ def compress(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self - def apply(self, bbuf, write, lbound_offset=0, size=0): - """Apply the chain's changes and write the final result using the passed - write function. - :param bbuf: base buffer containing the base of all deltas contained in this - list. It will only be used if the chunk in question does not have a base - chain. - :param lbound_offset: offset at which to start applying the delta, relative to - our lbound - :param size: if larger than 0, only the given amount of bytes will be applied - :param write: function taking a string of bytes to write to the output""" - slen = len(self) - if slen == 0: - return - # END early abort - absofs = self.lbound() + lbound_offset - if size == 0: - size = self.rbound() - absofs - # END initialize size - - dapply = DeltaChunk.apply - if lbound_offset or absofs + size != self.rbound(): - cdi = _closest_index(self, absofs) - cd = self[cdi] - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - dapply(tcd, bbuf, write) - size -= tcd.ts - cdi += 1 - # END handle first chunk - - # here we have to either apply full chunks, or smaller ones, but - # we always start at the chunks target offset - while cdi < slen and size: - cd = self[cdi] - if cd.ts <= size: - dapply(cd, bbuf, write) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - dapply(tcd, bbuf, write) - size -= tcd.ts - break - # END handle bytes to apply - cdi += 1 - # END handle rest - assert size == 0 - else: - for dc in self: - dapply(dc, bbuf, write) - # END for each dc - # END handle application values - def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -380,7 +385,6 @@ def __getslice__(self, absofs, size): # END hadle size cdi += 1 # END for each chunk - assert size == 0, "size was %i" % size # ndcl.check_integrity() return ndcl From e814fda2ef0b2de172fe06c2cbd1bc539a368262 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 12:04:59 +0200 Subject: [PATCH 0070/1790] First frame to implement the actual data aggregation, but ... its probbaly going to change quite a lot again --- fun.py | 46 +++++++++++++++++++++++++++++++--------------- stream.py | 2 +- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/fun.py b/fun.py index 253e64092..ec3662fe4 100644 --- a/fun.py +++ b/fun.py @@ -83,7 +83,7 @@ def _move_delta_lbound(d, bytes): return d def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data) + return DeltaChunk(src.to, src.ts, src.so, src.data, src.flags) def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer @@ -114,16 +114,18 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None + 'flags' # currently only True or False ) - def __init__(self, to, ts, so, data): + def __init__(self, to, ts, so, data, flags): self.to = to self.ts = ts self.so = so self.data = data + self.flags = flags def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + return "DeltaChunk(%i, %i, %s, %s, %i)" % (self.to, self.ts, self.so, self.data or "", self.flags) #{ Interface @@ -253,18 +255,24 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl): + def connect_with(self, bdcl, tdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base""" - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) - # END handle overlap - # END for each dc + :param bdcl: DeltaChunkList to serve as base + :param tdcl: topmost delta chunk list. If set, reverse order is assumed + and the list is connected more efficiently""" + if tdcl is None: + for dc in self: + if not dc.has_data(): + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + # END handle overlap + # END for each dc + else: + raise NotImplementedError("todo") + # END handle order def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines @@ -297,7 +305,7 @@ def compress(self): del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf, False)) slen = len(self) i = first_data_index + 1 @@ -522,14 +530,18 @@ def reverse_connect_deltas(dcl, dstreams): :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def connect_deltas(dstreams): +def connect_deltas(dstreams, reverse): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first + :param reverse: If False, the given iterable of delta-streams returns + items in from latest ancestor to the last delta. + If True, deltas are ordered so that the one to be applied last comes first. :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base dcl = DeltaChunkList() + tdcl = None # topmost dcl, only effective if reverse is True for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -576,12 +588,12 @@ def connect_deltas(dstreams): rbound > base_size): break - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None, False)) tbw += cp_size elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c], False)) i += c tbw += c else: @@ -591,9 +603,13 @@ def connect_deltas(dstreams): dcl.compress() + if reverse and tdcl is None: + tdcl = dcl + # END handle reverse + # merge the lists ! if bdcl is not None: - dcl.connect_with(bdcl) + dcl.connect_with(bdcl, tdcl) # END handle merge # dcl.check_integrity() diff --git a/stream.py b/stream.py index 16c917a97..40a0c6c6a 100644 --- a/stream.py +++ b/stream.py @@ -338,7 +338,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = connect_deltas(reversed(self._dstreams)) + dcl = connect_deltas(self._dstreams, reverse=True) if len(dcl) == 0: self._size = 0 From 2e19424354d03a5351ac374dc8dd36f6b65b0d5e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 12:33:14 +0200 Subject: [PATCH 0071/1790] New Frame uses a distinct type to express the different mode of operation. This is clean enough to get going --- fun.py | 85 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/fun.py b/fun.py index ec3662fe4..05925721c 100644 --- a/fun.py +++ b/fun.py @@ -237,7 +237,13 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): class DeltaChunkList(list): - """List with special functionality to deal with DeltaChunks""" + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() def rbound(self): """:return: rightmost extend in bytes, absolute""" @@ -255,24 +261,18 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl, tdcl): + def connect_with(self, bdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base - :param tdcl: topmost delta chunk list. If set, reverse order is assumed - and the list is connected more efficiently""" - if tdcl is None: - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) - # END handle overlap - # END for each dc - else: - raise NotImplementedError("todo") - # END handle order + :param bdcl: DeltaChunkList to serve as base""" + for dc in self: + if not dc.has_data(): + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + # END handle overlap + # END for each dc def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines @@ -396,8 +396,37 @@ def __getslice__(self, absofs, size): # ndcl.check_integrity() return ndcl - - + + +class TopdownDeltaChunkList(DeltaChunkList): + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = ('frozen', ) # if True, the list is frozen and can reproduce all data + # Will only be set in lists which where processed top-down + + def __init__(self): + self.frozen = False + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consequtively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams""" + if self.frozen == 1: + # Can that ever be hit ? + return False + # END early abort + # mark us so that the is_reversed method returns True, without us thinking + # we are frozen + self.frozen = -1 + + raise NotImplementedError("todo") + return True + #} END structures @@ -540,8 +569,14 @@ def connect_deltas(dstreams, reverse): If True, deltas are ordered so that the one to be applied last comes first. :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base - dcl = DeltaChunkList() tdcl = None # topmost dcl, only effective if reverse is True + + if reverse: + dcl = tdcl = TopdownDeltaChunkList() + else: + dcl = DeltaChunkList() + # END handle type of first chunk list + for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -603,13 +638,14 @@ def connect_deltas(dstreams, reverse): dcl.compress() - if reverse and tdcl is None: - tdcl = dcl - # END handle reverse - # merge the lists ! if bdcl is not None: - dcl.connect_with(bdcl, tdcl) + if tdcl: + if not tdcl.connect_with_next_base(dcl): + break + # END early abort + else: + dcl.connect_with(bdcl) # END handle merge # dcl.check_integrity() @@ -619,7 +655,10 @@ def connect_deltas(dstreams, reverse): dcl = DeltaChunkList() # END for each delta stream - return bdcl + if tdcl: + return tdcl + else: + return bdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): From f6bd67ce92257c9b5191b58de960cedda5159778 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 14:56:07 +0200 Subject: [PATCH 0072/1790] Reverse delta aggregration appears to be working --- fun.py | 155 +++++++++++++++++++++++++++++++--------------- test/test_pack.py | 2 +- 2 files changed, 107 insertions(+), 50 deletions(-) diff --git a/fun.py b/fun.py index 05925721c..9e4671a09 100644 --- a/fun.py +++ b/fun.py @@ -235,6 +235,46 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): # END for each dc # END handle application values +def delta_list_slice(dcl, absofs, size): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: DeltaChunkList (copy) which represents the given chunk""" + if len(dcl) == 0: + return DeltaChunkList() + + absofs = max(absofs, dcl.lbound()) + size = min(dcl.rbound() - dcl.lbound(), size) + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + ndcl = dcl.__class__() + + if cd.to != absofs: + tcd = delta_duplicate(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + ndcl.append(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + ndcl.append(delta_duplicate(cd)) + size -= cd.ts + else: + tcd = delta_duplicate(cd) + _set_delta_rbound(tcd, size) + ndcl.append(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + return ndcl class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. @@ -270,7 +310,7 @@ def connect_with(self, bdcl): :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts)) # END handle overlap # END for each dc @@ -355,49 +395,7 @@ def check_integrity(self, target_size=-1): assert lft.to + lft.ts == rgt.to # END for each pair - def __getslice__(self, absofs, size): - """:return: Subsection of this list at the given absolute offset, with the given - size in bytes. - :return: DeltaChunkList (copy) which represents the given chunk""" - if len(self) == 0: - return DeltaChunkList() - - absofs = max(absofs, self.lbound()) - size = min(self.rbound() - self.lbound(), size) - cdi = _closest_index(self, absofs) # delta start index - cd = self[cdi] - slen = len(self) - ndcl = self.__class__() - - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - ndcl.append(tcd) - size -= tcd.ts - cdi += 1 - # END lbound overlap handling - - while cdi < slen and size: - # are we larger than the current block - cd = self[cdi] - if cd.ts <= size: - ndcl.append(delta_duplicate(cd)) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - ndcl.append(tcd) - size -= tcd.ts - break - # END hadle size - cdi += 1 - # END for each chunk - - # ndcl.check_integrity() - return ndcl - class TopdownDeltaChunkList(DeltaChunkList): """Represents a list which is generated by feeding its ancestor streams one by one""" @@ -416,15 +414,76 @@ def connect_with_next_base(self, bdcl): consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams""" + assert self is not bdcl if self.frozen == 1: # Can that ever be hit ? return False # END early abort - # mark us so that the is_reversed method returns True, without us thinking - # we are frozen - self.frozen = -1 - raise NotImplementedError("todo") + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + sold = slen + while dci < slen: + dc = self[dci] + dci += 1 + + if dc.flags: + nfc += 1 + continue + # END skip frozen chunks + + # all data chunks must be frozen, we are topmost already + # (Also if its a copy operation onto the lowest base, but we cannot + # determine that without the number of deltas to come) + if dc.has_data(): + dc.flags = True + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + ccl = delta_list_slice(bdcl, dc.so, dc.ts) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + + assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound() + + if len(ccl) == 1: + self[dci-1] = ccl[0] + else: + + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci-1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl)-1 # deleted dc, added rest + + # END handle chunk replacement + + # END for each chunk + + if nfc == slen: + self.frozen = True + return False + # END handle completeness + return True @@ -648,8 +707,6 @@ def connect_deltas(dstreams, reverse): dcl.connect_with(bdcl) # END handle merge - # dcl.check_integrity() - # prepare next base bdcl = dcl dcl = DeltaChunkList() diff --git a/test/test_pack.py b/test/test_pack.py index 6e598d755..770a78bad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def _test_pack_entity(self): + def test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): From 0381cae63284e237a7023e9d40c7772690a2f84e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:11:02 +0200 Subject: [PATCH 0073/1790] Removed debugging code --- fun.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/fun.py b/fun.py index 9e4671a09..814c22a6a 100644 --- a/fun.py +++ b/fun.py @@ -399,11 +399,7 @@ def check_integrity(self, target_size=-1): class TopdownDeltaChunkList(DeltaChunkList): """Represents a list which is generated by feeding its ancestor streams one by one""" - __slots__ = ('frozen', ) # if True, the list is frozen and can reproduce all data - # Will only be set in lists which where processed top-down - - def __init__(self): - self.frozen = False + __slots__ = tuple() def connect_with_next_base(self, bdcl): """Connect this chain with the next level of our base delta chunklist. @@ -413,17 +409,10 @@ def connect_with_next_base(self, bdcl): :param bdcl: data chunk list being one of our bases. They must be fed in consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of - remaining streams""" - assert self is not bdcl - if self.frozen == 1: - # Can that ever be hit ? - return False - # END early abort - + remaining streams if False is returned""" nfc = 0 # number of frozen chunks dci = 0 # delta chunk index slen = len(self) # len of self - sold = slen while dci < slen: dc = self[dci] dci += 1 @@ -456,9 +445,6 @@ def connect_with_next_base(self, bdcl): cdc.to += ofs # END update target bounds - - assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound() - if len(ccl) == 1: self[dci-1] = ccl[0] else: @@ -476,14 +462,11 @@ def connect_with_next_base(self, bdcl): dci += len(ccl)-1 # deleted dc, added rest # END handle chunk replacement - # END for each chunk if nfc == slen: - self.frozen = True return False # END handle completeness - return True From 9b0773b92df4b4a2a53497efc9c1489028d403d7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:32:14 +0200 Subject: [PATCH 0074/1790] Removed previous non-reverse delta-application functionality. Although it was slightly faster, this new version only needs a faster slicing, which consumes ridiculous amounts of time --- fun.py | 134 ++++++++++++------------------------------------------ stream.py | 9 ++-- 2 files changed, 32 insertions(+), 111 deletions(-) diff --git a/fun.py b/fun.py index 814c22a6a..4d5edda9c 100644 --- a/fun.py +++ b/fun.py @@ -62,7 +62,6 @@ def _set_delta_rbound(d, size): # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE - return d def _move_delta_lbound(d, bytes): @@ -76,14 +75,14 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes d.ts -= bytes - if d.has_data(): + if d.data is not None: d.data = d.data[bytes:] # END handle data return d def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data, src.flags) + return DeltaChunk(src.to, src.ts, src.so, src.data) def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer @@ -92,8 +91,6 @@ def delta_chunk_apply(dc, bbuf, write): if dc.data is None: # COPY DATA FROM SOURCE write(buffer(bbuf, dc.so, dc.ts)) - elif isinstance(dc.data, DeltaChunkList): - delta_list_apply(dc.data, bbuf, write, dc.so, dc.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -105,6 +102,7 @@ def delta_chunk_apply(dc, bbuf, write): # END handle truncation # END handle chunk mode + class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -114,51 +112,25 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - 'flags' # currently only True or False ) - def __init__(self, to, ts, so, data, flags): + def __init__(self, to, ts, so, data): self.to = to self.ts = ts self.so = so self.data = data - self.flags = flags def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s, %i)" % (self.to, self.ts, self.so, self.data or "", self.flags) + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") #{ Interface - def copy_offset(self): - """:return: offset to apply when copying from a base buffer, or 0 - if this is not a copying delta chunk""" - - if self.data is not None: - if isinstance(self.data, DeltaChunkList): - return self.data.lbound() + self.so - else: - return self.so - # END handle data type - return 0 - def rbound(self): return self.to + self.ts def has_data(self): """:return: True if the instance has data to add to the target stream""" - return self.data is not None and not isinstance(self.data, DeltaChunkList) - - def has_copy_chunklist(self): - """:return: True if we copy our data from a chunklist""" - return self.data is not None and isinstance(self.data, DeltaChunkList) - - def set_copy_chunklist(self, dcl): - """Set the deltachunk list to be used as basis for copying. - :note: only works if this chunk is a copy delta chunk""" - self.data = dcl - self.so = 0 # allows lbound moves to be virtual - - + return self.data is not None #} END interface @@ -239,21 +211,20 @@ def delta_list_slice(dcl, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: DeltaChunkList (copy) which represents the given chunk""" - if len(dcl) == 0: - return DeltaChunkList() - - absofs = max(absofs, dcl.lbound()) - size = min(dcl.rbound() - dcl.lbound(), size) + dcllbound = dcl.lbound() + absofs = max(absofs, dcllbound) + size = min(dcl.rbound() - dcllbound, size) cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = dcl.__class__() + ndcl = DeltaChunkList() + lappend = ndcl.append if cd.to != absofs: tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) - ndcl.append(tcd) + lappend(tcd) size -= tcd.ts cdi += 1 # END lbound overlap handling @@ -262,12 +233,12 @@ def delta_list_slice(dcl, absofs, size): # are we larger than the current block cd = dcl[cdi] if cd.ts <= size: - ndcl.append(delta_duplicate(cd)) + lappend(delta_duplicate(cd)) size -= cd.ts else: tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) - ndcl.append(tcd) + lappend(tcd) size -= tcd.ts break # END hadle size @@ -301,19 +272,6 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl): - """Connect this instance's delta chunks virtually with the given base. - This means that all copy deltas will simply apply to the given region - of the given base. Afterwards, the base is optimized so that add-deltas - will be truncated to the region actually used, or removed completely where - adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base""" - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts)) - # END handle overlap - # END for each dc - def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines for performance""" @@ -333,7 +291,7 @@ def compress(self): while i < slen: dc = self[i] i += 1 - if not dc.has_data(): + if dc.data is None: if first_data_index is not None and i-2-first_data_index > 1: #if first_data_index is not None: nd = StringIO() # new data @@ -345,7 +303,7 @@ def compress(self): del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf, False)) + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) slen = len(self) i = first_data_index + 1 @@ -381,8 +339,6 @@ def check_integrity(self, target_size=-1): assert dc.ts > 0 if dc.has_data(): assert len(dc.data) >= dc.ts - if dc.has_copy_chunklist(): - assert dc.ts <= dc.data.size() # END for each dc left = islice(self, 0, len(self)-1) @@ -417,16 +373,8 @@ def connect_with_next_base(self, bdcl): dc = self[dci] dci += 1 - if dc.flags: - nfc += 1 - continue - # END skip frozen chunks - - # all data chunks must be frozen, we are topmost already - # (Also if its a copy operation onto the lowest base, but we cannot - # determine that without the number of deltas to come) - if dc.has_data(): - dc.flags = True + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: nfc += 1 continue # END skip add chunks @@ -448,7 +396,6 @@ def connect_with_next_base(self, bdcl): if len(ccl) == 1: self[dci-1] = ccl[0] else: - # maybe try to compute the expenses here, and pick the right algorithm # It would normally be faster than copying everything physically though # TODO: Use a deque here, and decide by the index whether to extend @@ -592,33 +539,16 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw -def reverse_connect_deltas(dcl, dstreams): +def connect_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: see 3 - :param dstreams: iterable of delta stream objects. They must be ordered latest first, - hence the delta to be applied last comes first, then its ancestors - :return: None""" - raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") - -def connect_deltas(dstreams, reverse): - """Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks - :param dstreams: iterable of delta stream objects. They must be ordered latest last, - hence the delta to be applied last comes last, its oldest ancestor first - :param reverse: If False, the given iterable of delta-streams returns - items in from latest ancestor to the last delta. - If True, deltas are ordered so that the one to be applied last comes first. + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base - tdcl = None # topmost dcl, only effective if reverse is True - - if reverse: - dcl = tdcl = TopdownDeltaChunkList() - else: - dcl = DeltaChunkList() - # END handle type of first chunk list + tdcl = None # topmost dcl + dcl = tdcl = TopdownDeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -665,12 +595,12 @@ def connect_deltas(dstreams, reverse): rbound > base_size): break - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None, False)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c], False)) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -682,12 +612,8 @@ def connect_deltas(dstreams, reverse): # merge the lists ! if bdcl is not None: - if tdcl: - if not tdcl.connect_with_next_base(dcl): - break - # END early abort - else: - dcl.connect_with(bdcl) + if not tdcl.connect_with_next_base(dcl): + break # END handle merge # prepare next base @@ -695,11 +621,7 @@ def connect_deltas(dstreams, reverse): dcl = DeltaChunkList() # END for each delta stream - if tdcl: - return tdcl - else: - return bdcl - + return tdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ diff --git a/stream.py b/stream.py index 40a0c6c6a..5292ce512 100644 --- a/stream.py +++ b/stream.py @@ -328,17 +328,16 @@ def __init__(self, stream_list): def _set_cache_(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python - # hence we apply a worst-case scenario here - # TODO: read the final size from the deltastream - have to partly unpack - # if len(self._dstreams) * self._size < self.k_max_memory_move: + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = connect_deltas(self._dstreams, reverse=True) + dcl = connect_deltas(self._dstreams) if len(dcl) == 0: self._size = 0 From 3837806673b992c1c0cb64203b50d9f1054e44db Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:45:54 +0200 Subject: [PATCH 0075/1790] Improved performance of delta_chunk_slice method a tiny bit, but it really needs to go to c --- fun.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fun.py b/fun.py index 4d5edda9c..75ff800be 100644 --- a/fun.py +++ b/fun.py @@ -210,14 +210,14 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): def delta_list_slice(dcl, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. - :return: DeltaChunkList (copy) which represents the given chunk""" + :return: list (copy) which represents the given chunk""" dcllbound = dcl.lbound() absofs = max(absofs, dcllbound) size = min(dcl.rbound() - dcllbound, size) cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = DeltaChunkList() + ndcl = list() lappend = ndcl.append if cd.to != absofs: From 89408f8ec351c1d14f66caf53a2c1e163bde0f27 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 23:08:57 +0200 Subject: [PATCH 0076/1790] Initial frame of the connect_delta method, which seems to do something. Debugging is hellish, you really have to use python exception to get information out of there, printf doesn't do anything for some reason --- Makefile | 24 +++++ _fun.c | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++++- fun.py | 5 + stream.py | 5 +- 4 files changed, 310 insertions(+), 3 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..190a66b03 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +PYTHON = python +SETUP = $(PYTHON) setup.py +TESTRUNNER = $(shell which nosetests) +TESTFLAGS = + +all: build + +doc:: + make -C doc/ html + +build:: + $(SETUP) build + $(SETUP) build_ext -i + +install:: + $(SETUP) install + +clean:: + $(SETUP) clean --all + rm -f *.so + +coverage:: build + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=dulwich --with-coverage --cover-erase --cover-inclusive gitdb + diff --git a/_fun.c b/_fun.c index ce9f25b16..e9f769daa 100644 --- a/_fun.c +++ b/_fun.c @@ -1,5 +1,7 @@ #include #include +#include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -82,16 +84,289 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) } +typedef unsigned long long ull; + +// Internal Delta Chunk Objects +typedef struct { + ull to; + ull ts; + ull so; + PyObject* data; + + void* next; +} DeltaChunk; + + +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data, DeltaChunk* next) +{ + dc->to = to; + dc->ts = ts; + dc->so = so; + Py_XINCREF(data); + dc->data = data; + + dc->next = next; +} + +void DC_destroy(DeltaChunk* dc) +{ + Py_XDECREF(dc->data); +} + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunk* head; + DeltaChunk* tail; + ull size; + +} DeltaChunkList; + +ull DC_rbound(DeltaChunk* dc) +{ + return dc->to + dc->ts; +} + + +static +int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) +{ + ((DeltaChunkList*)self)->head = NULL; + return 1; +} + +static +void DCL_dealloc(DeltaChunkList* self) +{ + // TODO: deallocate linked list + if (self->head){ + self->head = NULL; + self->tail = NULL; + self->size = 0; + } +} + +static +PyObject* DCL_len(PyObject* self) +{ + return PyLong_FromUnsignedLongLong(0); +} + +static +PyObject* DCL_rbound(DeltaChunkList* self) +{ + if (!self->head) + return PyLong_FromUnsignedLongLong(0); + return PyLong_FromUnsignedLongLong(DC_rbound(self->tail)); +} + +static +PyObject* DCL_apply(PyObject* self, PyObject* args) +{ + + Py_RETURN_NONE; +} + + + +static PyMethodDef DCL_methods[] = { + {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, + {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_rbound, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject DeltaChunkListType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "DeltaChunkList", /*tp_name*/ + sizeof(DeltaChunkList), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)DCL_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "Minimal Delta Chunk List",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DCL_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)DCL_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static inline +ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ + ull size = 0; + Py_ssize_t i = 0; + const char* dend = data + dlen; + for (data = data + offset; data < dend; data+=1, i+=1){ + char c = *data; + size |= (c & 0x7f) << i*7; + if (!(c & 0x80)){ + break; + } + }// END while in range + + *out_bytes_read = i+offset; + return size; +} + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +{ + // obtain iterator + PyObject* stream_iter = 0; + if (!PyIter_Check(dstreams)){ + stream_iter = PyObject_GetIter(dstreams); + if (!stream_iter){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); + return NULL; + } + } else { + stream_iter = dstreams; + } + + DeltaChunkList* bdcl = 0; + DeltaChunkList* tdcl = 0; + DeltaChunkList* dcl = 0; + + dcl = tdcl = PyObject_New(DeltaChunkList, &DeltaChunkListType); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + return NULL; + } + + unsigned int dsi; + PyObject* ds; + int error = 0; + for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + { + PyObject* db = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(db)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); + goto loop_end; + } + + const char* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + + // read header + Py_ssize_t ofs = 0; + const ull base_size = msb_size(data, dlen, 0, &ofs); + const ull target_size = msb_size(data, dlen, ofs, &ofs); + + // parse command stream + const char* dend = data + dlen; + ull tbw = 0; // Amount of target bytes written + for (data = data + ofs; data < dend; ++data) + { + const char cmd = *data; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + const unsigned long rbound = cp_off + cp_size; + if (rbound < cp_size || + rbound > base_size){ + goto loop_end; + } + + // TODO: Add node + tbw += cp_size; + + } else if (cmd) { + // TODO: Add node + tbw += cmd; + } else { + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + goto loop_end; + } + }// END handle command opcodes + + assert(tbw == target_size); + +loop_end: + // perform cleanup + Py_DECREF(ds); + Py_DECREF(db); + + if (error){ + break; + } + }// END for each stream object + + if (dsi == 0 && ! error){ + PyErr_SetString(PyExc_ValueError, "No streams provided"); + } + + if (stream_iter != dstreams){ + Py_DECREF(stream_iter); + } + + if (error){ + return NULL; + } + + return (PyObject*)tdcl; +} + static PyMethodDef py_fun[] = { - { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL }, + { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; -void init_fun(void) +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC init_fun(void) { PyObject *m; + DeltaChunkListType.tp_new = PyType_GenericNew; + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + m = Py_InitModule3("_fun", py_fun, NULL); if (m == NULL) return; + + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "Noddy", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index 75ff800be..a4da30903 100644 --- a/fun.py +++ b/fun.py @@ -701,3 +701,8 @@ def is_equal_canonical_sha(canonical_length, match, sha1): #} END routines + +try: + from _fun import connect_deltas +except ImportError: + pass diff --git a/stream.py b/stream.py index 5292ce512..169104625 100644 --- a/stream.py +++ b/stream.py @@ -338,8 +338,11 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = connect_deltas(self._dstreams) + assert dcl is not None - if len(dcl) == 0: + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.__len__() == 0: self._size = 0 self._mm_target = allocate_memory(0) return From 2409fafca6f0518500571bcf82e92554f2c50b85 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 09:19:12 +0200 Subject: [PATCH 0077/1790] Implemented a few more functions, but I realize the vector implementation actually wants to be in a separate structure --- _fun.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/_fun.c b/_fun.c index e9f769daa..8abc5bcbb 100644 --- a/_fun.c +++ b/_fun.c @@ -116,9 +116,9 @@ void DC_destroy(DeltaChunk* dc) typedef struct { PyObject_HEAD // ----------- - DeltaChunk* head; - DeltaChunk* tail; + DeltaChunk* mem; ull size; + ull reserved_size; } DeltaChunkList; @@ -127,22 +127,65 @@ ull DC_rbound(DeltaChunk* dc) return dc->to + dc->ts; } +static +int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) +{ + self->mem = NULL; // Memory + self->size = 0; // Size in DeltaChunks + self->reserved_size = 0; // Reserve in DeltaChunks + return 1; +} + +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +static +int DCL_grow(DeltaChunkList* self, ull num_dc) +{ + const ull grow_by_chunks = (self->size + num_dc) - self->reserved_size; + if (grow_by_chunks <= 0){ + return 1; + } + + if (self->mem){ + self->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + } else { + self->mem = PyMem_Realloc(self->mem, (self->size + grow_by_chunks)*sizeof(DeltaChunk)); + } + + return self->mem != NULL; +} + static int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) { - ((DeltaChunkList*)self)->head = NULL; - return 1; + if(PySequence_Size(args) > 1){ + PyErr_SetString(PyExc_ValueError, "Zero or one arguments are allowed, providing the initial size of the queue in DeltaChunks"); + return 0; + } + + ull init_size = 0; + PyArg_ParseTuple(args, "K", &init_size); + if (init_size == 0){ + init_size = 125000; + } + + return DCL_grow(self, init_size); } static void DCL_dealloc(DeltaChunkList* self) { // TODO: deallocate linked list - if (self->head){ - self->head = NULL; - self->tail = NULL; + if (self->mem){ + PyMem_Free(self->mem); self->size = 0; + self->reserved_size = 0; + self->mem = 0; } } @@ -153,11 +196,18 @@ PyObject* DCL_len(PyObject* self) } static -PyObject* DCL_rbound(DeltaChunkList* self) +inline +ull DCL_rbound(DeltaChunkList* self) +{ + if (!self->mem | !self->size) + return 0; + return DC_rbound(&(self->mem[self->size-1])); +} + +static +PyObject* DCL_py_rbound(DeltaChunkList* self) { - if (!self->head) - return PyLong_FromUnsignedLongLong(0); - return PyLong_FromUnsignedLongLong(DC_rbound(self->tail)); + return PyLong_FromUnsignedLongLong(DCL_rbound(self)); } static @@ -172,7 +222,7 @@ PyObject* DCL_apply(PyObject* self, PyObject* args) static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, - {"rbound", (PyCFunction)DCL_rbound, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -215,7 +265,7 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_dictoffset */ (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + (newfunc)DCL_new, /* tp_new */ }; @@ -233,6 +283,7 @@ ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* o }// END while in range *out_bytes_read = i+offset; + assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull)); return size; } From 511a29dab7c7a507abc3bc656b5e9f5ab2db85a3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 09:54:15 +0200 Subject: [PATCH 0078/1790] DeltaChunkVector is now a separate structure. I wished I had c++, but ... its probably a good exercise --- _fun.c | 126 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/_fun.c b/_fun.c index 8abc5bcbb..fb83f26a6 100644 --- a/_fun.c +++ b/_fun.c @@ -85,27 +85,26 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; +typedef unsigned int uint; + +// DELTA CHUNK +//////////////// // Internal Delta Chunk Objects typedef struct { ull to; ull ts; ull so; PyObject* data; - - void* next; } DeltaChunk; - -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data, DeltaChunk* next) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data) { dc->to = to; dc->ts = ts; dc->so = so; Py_XINCREF(data); dc->data = data; - - dc->next = next; } void DC_destroy(DeltaChunk* dc) @@ -113,28 +112,20 @@ void DC_destroy(DeltaChunk* dc) Py_XDECREF(dc->data); } -typedef struct { - PyObject_HEAD - // ----------- - DeltaChunk* mem; - ull size; - ull reserved_size; - -} DeltaChunkList; - ull DC_rbound(DeltaChunk* dc) { return dc->to + dc->ts; } -static -int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) -{ - self->mem = NULL; // Memory - self->size = 0; // Size in DeltaChunks - self->reserved_size = 0; // Reserve in DeltaChunks - return 1; -} + +// DELTA CHUNK VECTOR +///////////////////// + +typedef struct { + DeltaChunk* mem; // Memory + Py_ssize_t size; // Size in DeltaChunks + Py_ssize_t reserved_size; // Reserve in DeltaChunks +} DeltaChunkVector; /* Grow the delta chunk list by the given amount of bytes. @@ -143,20 +134,76 @@ large enough. Return 1 on success, 0 on failure */ static -int DCL_grow(DeltaChunkList* self, ull num_dc) +int DCV_grow(DeltaChunkVector* vec, uint num_dc) { - const ull grow_by_chunks = (self->size + num_dc) - self->reserved_size; + const ull grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; if (grow_by_chunks <= 0){ return 1; } - if (self->mem){ - self->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + if (vec->mem){ + vec->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); } else { - self->mem = PyMem_Realloc(self->mem, (self->size + grow_by_chunks)*sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, (vec->size + grow_by_chunks)*sizeof(DeltaChunk)); } - return self->mem != NULL; + return vec->mem != NULL; +} + +int DCV_init(DeltaChunkVector* vec, ull initial_size) +{ + vec->mem = NULL; + vec->size = 0; + vec->reserved_size = 0; + + return DCV_grow(vec, initial_size); +} + + +void DCV_dealloc(DeltaChunkVector* vec) +{ + if (vec->mem){ + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +static inline +ull DCV_len(DeltaChunkVector* vec) +{ + return vec->size; +} + +// Return item at index +static inline +DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) +{ + assert(i < vec->size && vec->mem); + return &(vec->mem[i]); +} + +static inline +int DCV_empty(DeltaChunkVector* vec) +{ + return vec->size == 0; +} + +// DELTA CHUNK LIST (PYTHON) +///////////////////////////// + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunkVector vec; + +} DeltaChunkList; + +static +int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) +{ + return DCV_init(&self->vec, 0); } @@ -174,34 +221,27 @@ int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) init_size = 125000; } - return DCL_grow(self, init_size); + return DCV_grow(&self->vec, init_size); } static void DCL_dealloc(DeltaChunkList* self) { - // TODO: deallocate linked list - if (self->mem){ - PyMem_Free(self->mem); - self->size = 0; - self->reserved_size = 0; - self->mem = 0; - } + DCV_dealloc(&self->vec); } static -PyObject* DCL_len(PyObject* self) +PyObject* DCL_len(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(0); + return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); } -static -inline +static inline ull DCL_rbound(DeltaChunkList* self) { - if (!self->mem | !self->size) + if (DCV_empty(&self->vec)) return 0; - return DC_rbound(&(self->mem[self->size-1])); + return DC_rbound(DCV_get(&self->vec, self->vec.size - 1)); } static From f030fa1d7ef6a43cc05b11e3be2108f83d9683f9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 11:25:23 +0200 Subject: [PATCH 0079/1790] Weird bug causes crash, its memory related of course. GDB tells me where, but the why is still a mystery --- _fun.c | 82 +++++++++++++++++++++++++++++++++++++------------------ stream.py | 3 ++ 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/_fun.c b/_fun.c index fb83f26a6..47f5d8ecf 100644 --- a/_fun.c +++ b/_fun.c @@ -136,16 +136,18 @@ Return 1 on success, 0 on failure static int DCV_grow(DeltaChunkVector* vec, uint num_dc) { - const ull grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; + const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; if (grow_by_chunks <= 0){ return 1; } - if (vec->mem){ - vec->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + if (vec->mem == NULL){ + vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(vec->mem)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->size + grow_by_chunks)*sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(vec->mem)); } + assert(vec->mem != NULL); + vec->reserved_size = vec->reserved_size + grow_by_chunks; return vec->mem != NULL; } @@ -159,17 +161,6 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) return DCV_grow(vec, initial_size); } - -void DCV_dealloc(DeltaChunkVector* vec) -{ - if (vec->mem){ - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - static inline ull DCV_len(DeltaChunkVector* vec) { @@ -181,7 +172,7 @@ static inline DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); - return &(vec->mem[i]); + return &vec->mem[i]; } static inline @@ -190,6 +181,48 @@ int DCV_empty(DeltaChunkVector* vec) return vec->size == 0; } +// Return end pointer of the vector +static inline +DeltaChunk* DCV_end(DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return &vec->mem[vec->size]; +} + +void DCV_dealloc(DeltaChunkVector* vec) +{ + if (vec->mem){ + if (vec->size){ + const DeltaChunk* end = DCV_end(vec); + DeltaChunk* i; + for(i = &vec->mem[0]; i < end; i++){ + DC_destroy(i); + } + } + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +// Append num-chunks to the end of the list, possibly reallocating existing ones +// Return a pointer to the first of the added items. They are not yet initialized +// If num-chunks == 0, it returns the end pointer of the allocated memory +static inline +DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) +{ + if (vec->size + num_chunks > vec->reserved_size){ + if (!DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size)){ + Py_FatalError("Could not allocate memory for append operation"); + } + } + Py_FatalError("Could not allocate memory for append operation"); + Py_ssize_t old_size = vec->size; + vec->size += num_chunks; + return &vec->mem[old_size]; +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -200,12 +233,6 @@ typedef struct { } DeltaChunkList; -static -int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) -{ - return DCV_init(&self->vec, 0); -} - static int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) @@ -215,19 +242,20 @@ int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) return 0; } + assert(self->vec.mem == NULL); + ull init_size = 0; PyArg_ParseTuple(args, "K", &init_size); if (init_size == 0){ - init_size = 125000; + init_size = 12500; } - - return DCV_grow(&self->vec, init_size); + return DCV_init(&self->vec, init_size); } static void DCL_dealloc(DeltaChunkList* self) { - DCV_dealloc(&self->vec); + DCV_dealloc(&(self->vec)); } static @@ -305,7 +333,7 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_dictoffset */ (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - (newfunc)DCL_new, /* tp_new */ + 0, /* tp_new */ }; diff --git a/stream.py b/stream.py index 169104625..6d7479d1c 100644 --- a/stream.py +++ b/stream.py @@ -339,6 +339,9 @@ def _set_cache_(self, attr): # the final delta data stream. dcl = connect_deltas(self._dstreams) assert dcl is not None + print "got dcl" + del(dcl) + print "dealloc worked" # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 445b71637576649fd6f1f3c287f50eec24d4fbf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 12:43:57 +0200 Subject: [PATCH 0080/1790] Wow, this was a lesson. My full hatred goes to python, and C, and everything ... cool if you control everything, but not cool if an Object_New call doesn't do anything for you - creating a new instance of an own type in python doesn't appear to be that easy after all, at least not if you want your initializers/new procs to be called --- _fun.c | 60 +++++++++++++++++++++++++++++++------------------------ stream.py | 3 --- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/_fun.c b/_fun.c index 47f5d8ecf..3a1bd0355 100644 --- a/_fun.c +++ b/_fun.c @@ -192,13 +192,12 @@ DeltaChunk* DCV_end(DeltaChunkVector* vec) void DCV_dealloc(DeltaChunkVector* vec) { if (vec->mem){ - if (vec->size){ - const DeltaChunk* end = DCV_end(vec); - DeltaChunk* i; - for(i = &vec->mem[0]; i < end; i++){ - DC_destroy(i); - } + const DeltaChunk* end = DCV_end(vec); + DeltaChunk* i; + for(i = vec->mem; i < end; i++){ + DC_destroy(i); } + PyMem_Free(vec->mem); vec->size = 0; vec->reserved_size = 0; @@ -207,7 +206,7 @@ void DCV_dealloc(DeltaChunkVector* vec) } // Append num-chunks to the end of the list, possibly reallocating existing ones -// Return a pointer to the first of the added items. They are not yet initialized +// Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) @@ -220,6 +219,11 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; vec->size += num_chunks; + + for(;old_size < vec->size; ++old_size){ + DC_init(DCV_get(vec, old_size), 0, 0, 0, NULL); + } + return &vec->mem[old_size]; } @@ -235,21 +239,15 @@ typedef struct { static -int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) +int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) { - if(PySequence_Size(args) > 1){ - PyErr_SetString(PyExc_ValueError, "Zero or one arguments are allowed, providing the initial size of the queue in DeltaChunks"); - return 0; + if(args && PySequence_Size(args) > 0){ + PyErr_SetString(PyExc_ValueError, "Too many arguments"); + return -1; } - assert(self->vec.mem == NULL); - - ull init_size = 0; - PyArg_ParseTuple(args, "K", &init_size); - if (init_size == 0){ - init_size = 12500; - } - return DCV_init(&self->vec, init_size); + DCV_init(&self->vec, 0); + return 0; } static @@ -285,8 +283,6 @@ PyObject* DCL_apply(PyObject* self, PyObject* args) Py_RETURN_NONE; } - - static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, @@ -331,12 +327,24 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ + (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + 0, /* tp_new */ }; +// Makes a new copy of the DeltaChunkList - you have to do everything yourselve +// in C ... want C++ !! +DeltaChunkList* DCL_new_instance(void) +{ + DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); + assert(dcl); + + DCL_init(dcl, 0, 0); + assert(dcl->vec.size == 0); + return dcl; +} + static inline ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ ull size = 0; @@ -351,7 +359,7 @@ ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* o }// END while in range *out_bytes_read = i+offset; - assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull)); + assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull) * 8); return size; } @@ -373,7 +381,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkList* tdcl = 0; DeltaChunkList* dcl = 0; - dcl = tdcl = PyObject_New(DeltaChunkList, &DeltaChunkListType); + dcl = tdcl = DCL_new_instance(); + assert(dcl != NULL); if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); return NULL; @@ -478,7 +487,6 @@ PyMODINIT_FUNC init_fun(void) { PyObject *m; - DeltaChunkListType.tp_new = PyType_GenericNew; if (PyType_Ready(&DeltaChunkListType) < 0) return; diff --git a/stream.py b/stream.py index 6d7479d1c..169104625 100644 --- a/stream.py +++ b/stream.py @@ -339,9 +339,6 @@ def _set_cache_(self, attr): # the final delta data stream. dcl = connect_deltas(self._dstreams) assert dcl is not None - print "got dcl" - del(dcl) - print "dealloc worked" # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 9ba93c0deb4f46b524e28c8103b35f408a936e04 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 14:58:38 +0200 Subject: [PATCH 0081/1790] Apparently, the most serious memory bugs are fixed for now, lets get back to the actual thing --- _fun.c | 73 +++++++++++++++++++++++++++++++++++++++++++------------ stream.py | 1 - 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/_fun.c b/_fun.c index 3a1bd0355..7089cfe3d 100644 --- a/_fun.c +++ b/_fun.c @@ -2,6 +2,7 @@ #include #include #include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -87,7 +88,6 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; - // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -142,13 +142,17 @@ int DCV_grow(DeltaChunkVector* vec, uint num_dc) } if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(vec->mem)); + vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(DeltaChunk)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(vec->mem)); + vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); } assert(vec->mem != NULL); vec->reserved_size = vec->reserved_size + grow_by_chunks; +#ifdef DEBUG + fprintf(stderr, "Allocated %i bytes at %p, to hold up to %i chunks\n", (int)((vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)), vec->mem, (int)(vec->reserved_size + grow_by_chunks)); +#endif + return vec->mem != NULL; } @@ -192,7 +196,11 @@ DeltaChunk* DCV_end(DeltaChunkVector* vec) void DCV_dealloc(DeltaChunkVector* vec) { if (vec->mem){ - const DeltaChunk* end = DCV_end(vec); +#ifdef DEBUG + fprintf(stderr, "Freeing %p\n", (void*)vec->mem); +#endif + + const DeltaChunk* end = &vec->mem[vec->size]; DeltaChunk* i; for(i = vec->mem; i < end; i++){ DC_destroy(i); @@ -342,6 +350,7 @@ DeltaChunkList* DCL_new_instance(void) DCL_init(dcl, 0, 0); assert(dcl->vec.size == 0); + assert(dcl->vec.mem == NULL); return dcl; } @@ -377,16 +386,13 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkList* bdcl = 0; - DeltaChunkList* tdcl = 0; - DeltaChunkList* dcl = 0; + DeltaChunkVector bdcv; + DeltaChunkVector tdcv; + DeltaChunkVector dcv; + DCV_init(&bdcv, 0); + DCV_init(&dcv, 0); + DCV_init(&tdcv, 0); - dcl = tdcl = DCL_new_instance(); - assert(dcl != NULL); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - return NULL; - } unsigned int dsi; PyObject* ds; @@ -408,6 +414,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_ssize_t ofs = 0; const ull base_size = msb_size(data, dlen, 0, &ofs); const ull target_size = msb_size(data, dlen, ofs, &ofs); + + // estimate number of ops - assume one third adds, half two byte (size+offset) copies + const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + DCV_grow(&dcv, approx_num_cmds); // parse command stream const char* dend = data + dlen; @@ -431,7 +441,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ - goto loop_end; + break; } // TODO: Add node @@ -446,8 +456,19 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } }// END handle command opcodes - assert(tbw == target_size); + + // swap the vector + // Skip the first vector, as it is also used as top chunk vector + if (bdcv.mem != tdcv.mem){ + DCV_dealloc(&bdcv); + } + bdcv = dcv; + if (dsi == 0){ + tdcv = dcv; + } + DCV_init(&dcv, 0); + loop_end: // perform cleanup @@ -467,11 +488,31 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } + DCV_dealloc(&bdcv); + if (dsi > 1){ + // otherwise dcv equals tcl + DCV_dealloc(&dcv); + } + + // Return the actual python object - its just a container + DeltaChunkList* dcl = DCL_new_instance(); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + // Otherwise tdcv would be deallocated by the chunk list + DCV_dealloc(&tdcv); + error = 1; + } else { + // Plain copy, don't deallocate + dcl->vec = tdcv; + } + if (error){ + // Will dealloc tdcv + Py_XDECREF(dcl); return NULL; } - return (PyObject*)tdcl; + return (PyObject*)dcl; } static PyMethodDef py_fun[] = { diff --git a/stream.py b/stream.py index 169104625..af4591f85 100644 --- a/stream.py +++ b/stream.py @@ -338,7 +338,6 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = connect_deltas(self._dstreams) - assert dcl is not None # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 489f763308d4982a5220a648b92ecb2cc82f4ae4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 16:04:55 +0200 Subject: [PATCH 0082/1790] Now adding chunks to the vectors, next up is to implement the actual chunk merging --- _fun.c | 104 ++++++++++++++++++++++++++++++++++++++------------------- fun.py | 1 + 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/_fun.c b/_fun.c index 7089cfe3d..2ad8d641d 100644 --- a/_fun.c +++ b/_fun.c @@ -3,6 +3,7 @@ #include #include #include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -87,6 +88,7 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; +typedef unsigned char uchar; // DELTA CHUNK //////////////// @@ -95,21 +97,38 @@ typedef struct { ull to; ull ts; ull so; - PyObject* data; + uchar* data; } DeltaChunk; -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) { dc->to = to; dc->ts = ts; dc->so = so; - Py_XINCREF(data); - dc->data = data; + dc->data = NULL; } void DC_destroy(DeltaChunk* dc) { - Py_XDECREF(dc->data); + if (dc->data){ + PyMem_Free((void*)dc->data); + } +} + +// Store a copy of data in our instance +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen) +{ + if (dc->data){ + PyMem_Free((void*)dc->data); + } + + if (data == 0){ + dc->data = NULL; + return; + } + + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy(dc->data, data, dlen); } ull DC_rbound(DeltaChunk* dc) @@ -146,7 +165,11 @@ int DCV_grow(DeltaChunkVector* vec, uint num_dc) } else { vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); } - assert(vec->mem != NULL); + + if (vec->mem == NULL){ + Py_FatalError("Could not allocate memory for append operation"); + } + vec->reserved_size = vec->reserved_size + grow_by_chunks; #ifdef DEBUG @@ -220,21 +243,33 @@ static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) { if (vec->size + num_chunks > vec->reserved_size){ - if (!DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size)){ - Py_FatalError("Could not allocate memory for append operation"); - } + DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size); } Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; vec->size += num_chunks; for(;old_size < vec->size; ++old_size){ - DC_init(DCV_get(vec, old_size), 0, 0, 0, NULL); + DC_init(DCV_get(vec, old_size), 0, 0, 0); } return &vec->mem[old_size]; } +// Append one chunk to the end of the list, and return a pointer to it +// It will have been initialized. +static inline +DeltaChunk* DCV_append(DeltaChunkVector* vec) +{ + if (vec->size + 1 > vec->reserved_size){ + DCV_grow(vec, 1); + } + + DeltaChunk* next = vec->mem + vec->size; + vec->size += 1; + return next; +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -354,21 +389,18 @@ DeltaChunkList* DCL_new_instance(void) return dcl; } -static inline -ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ - ull size = 0; - Py_ssize_t i = 0; - const char* dend = data + dlen; - for (data = data + offset; data < dend; data+=1, i+=1){ - char c = *data; - size |= (c & 0x7f) << i*7; - if (!(c & 0x80)){ - break; - } - }// END while in range - - *out_bytes_read = i+offset; - assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull) * 8); +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; return size; } @@ -406,25 +438,25 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } - const char* data; + const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + const uchar* dend = data + dlen; // read header - Py_ssize_t ofs = 0; - const ull base_size = msb_size(data, dlen, 0, &ofs); - const ull target_size = msb_size(data, dlen, ofs, &ofs); + const ull base_size = msb_size(&data, dend); + const ull target_size = msb_size(&data, dend); // estimate number of ops - assume one third adds, half two byte (size+offset) copies const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); DCV_grow(&dcv, approx_num_cmds); // parse command stream - const char* dend = data + dlen; ull tbw = 0; // Amount of target bytes written - for (data = data + ofs; data < dend; ++data) + assert(data < dend); + while (data < dend) { - const char cmd = *data; + const char cmd = *data++; if (cmd & 0x80) { @@ -444,12 +476,16 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) break; } - // TODO: Add node + DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); tbw += cp_size; } else if (cmd) { - // TODO: Add node + // TODO: Compress nodes by parsing them in advance + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, cmd, 0); + DC_set_data(dc, data, cmd); tbw += cmd; + data += cmd; } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); diff --git a/fun.py b/fun.py index a4da30903..e6262b4bc 100644 --- a/fun.py +++ b/fun.py @@ -703,6 +703,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: + # raise ImportError; # DEBUG from _fun import connect_deltas except ImportError: pass From a93363cffb225520869d737de1081c1ff77ed108 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 17:45:42 +0200 Subject: [PATCH 0083/1790] prepared the slicing, as well as a few accompanying methods. There is still quite a lot functionality missing --- _fun.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 166 insertions(+), 28 deletions(-) diff --git a/_fun.c b/_fun.c index 2ad8d641d..1186c6d11 100644 --- a/_fun.c +++ b/_fun.c @@ -89,6 +89,7 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; +typedef uchar bool; // DELTA CHUNK //////////////// @@ -97,45 +98,96 @@ typedef struct { ull to; ull ts; ull so; - uchar* data; + const uchar* data; + bool data_shared; } DeltaChunk; +inline void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) { dc->to = to; dc->ts = ts; dc->so = so; dc->data = NULL; + dc->data_shared = 0; } -void DC_destroy(DeltaChunk* dc) +inline +void DC_deallocate_data(DeltaChunk* dc) { - if (dc->data){ + if (!dc->data_shared && dc->data){ PyMem_Free((void*)dc->data); } + dc->data = NULL; } -// Store a copy of data in our instance -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen) +inline +void DC_destroy(DeltaChunk* dc) { - if (dc->data){ - PyMem_Free((void*)dc->data); - } + DC_deallocate_data(dc); +} + +// Store a copy of data in our instance. If shared is 1, the data will be shared, +// hence it will only be stored, but the memory will not be touched, or copied. +inline +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) +{ + DC_deallocate_data(dc); if (data == 0){ dc->data = NULL; + dc->data_shared = 0; return; } - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy(dc->data, data, dlen); + dc->data_shared = shared; + if (shared){ + dc->data = data; + } else { + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy((void*)dc->data, (void*)data, dlen); + } + } +inline ull DC_rbound(DeltaChunk* dc) { return dc->to + dc->ts; } +// Copy all data from src to dest, the data pointer will be copied too +inline +void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) +{ + dest->to = src->to; + dest->ts = src->ts; + dest->so = src->so; + dest->data_shared = 0; + + DC_set_data(dest, src->data, src->ts, 0); +} + +// Copy all data with the given offset and size. The source offset, as well +// as the data will be truncated accordingly +inline +void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +{ + assert(size <= src->ts); + assert(src->to + ofs + size <= DC_rbound(src)); + + dest->to = src->to + ofs; + dest->ts = size; + dest->so = src->so + ofs; + + if (src->data){ + DC_set_data(dest, src->data + ofs, size, 0); + } else { + dest->data = NULL; + dest->data_shared = 0; + } +} + // DELTA CHUNK VECTOR ///////////////////// @@ -152,7 +204,7 @@ This may trigger a realloc, but will do nothing if the reserved size is already large enough. Return 1 on success, 0 on failure */ -static +inline int DCV_grow(DeltaChunkVector* vec, uint num_dc) { const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; @@ -188,35 +240,48 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) return DCV_grow(vec, initial_size); } -static inline +inline ull DCV_len(DeltaChunkVector* vec) { return vec->size; } +inline +ull DCV_lbound(DeltaChunkVector* vec) +{ + assert(vec->size && vec->mem); + return vec->mem->to; +} + // Return item at index -static inline +inline DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; } -static inline +inline +ull DCV_rbound(DeltaChunkVector* vec) +{ + return DC_rbound(DCV_get(vec, vec->size-1)); +} + +inline int DCV_empty(DeltaChunkVector* vec) { return vec->size == 0; } // Return end pointer of the vector -static inline +inline DeltaChunk* DCV_end(DeltaChunkVector* vec) { assert(!DCV_empty(vec)); return &vec->mem[vec->size]; } -void DCV_dealloc(DeltaChunkVector* vec) +void DCV_destroy(DeltaChunkVector* vec) { if (vec->mem){ #ifdef DEBUG @@ -236,6 +301,14 @@ void DCV_dealloc(DeltaChunkVector* vec) } } +// Reset this vector so that its existing memory can be filled again. +// Memory will be kept, but not cleaned up +inline +void DCV_forget_members(DeltaChunkVector* vec) +{ + vec->size = 0; +} + // Append num-chunks to the end of the list, possibly reallocating existing ones // Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory @@ -249,15 +322,17 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_ssize_t old_size = vec->size; vec->size += num_chunks; +#ifdef DEBUG for(;old_size < vec->size; ++old_size){ DC_init(DCV_get(vec, old_size), 0, 0, 0); } +#endif return &vec->mem[old_size]; } // Append one chunk to the end of the list, and return a pointer to it -// It will have been initialized. +// It will not have been initialized ! static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { @@ -270,6 +345,59 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) return next; } +// Write a slice as defined by its absolute offset in bytes and its size into the given +// destination. The individual chunks written will be a deep copy of the source +// data chunks +// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort +// of append-only memory pool would improve performance +inline +void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +{ + +} + + +// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost +// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the +// caller +static +void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +{ + DeltaChunk* dc = tdcv->mem; + DeltaChunk* end = tdcv->mem + tdcv->size; + assert(dc); + + for (;dc < end; dc++) + { + // Data chunks don't need processing + if (dc->data){ + continue; + } + + // Copy Chunk Handling + DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); + // assert(tmpl->size); + + // move target bounds + DeltaChunk* cdc = tmpl->mem; + DeltaChunk* cdcend = tmpl->mem + tmpl->size; + const ull ofs = dc->to - dc->so; + for(;cdc < cdcend; cdc++){ + cdc->to += ofs; + } + + // insert slice into our list, replacing our current chunk + if (tmpl->size == 1){ + *dc = *DCV_get(tmpl, 0); + } else { + + } + + // make sure the members will not be deallocated by the list + DCV_forget_members(tmpl); + } +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -296,7 +424,7 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) static void DCL_dealloc(DeltaChunkList* self) { - DCV_dealloc(&(self->vec)); + DCV_destroy(&(self->vec)); } static @@ -310,7 +438,7 @@ ull DCL_rbound(DeltaChunkList* self) { if (DCV_empty(&self->vec)) return 0; - return DC_rbound(DCV_get(&self->vec, self->vec.size - 1)); + return DCV_rbound(&self->vec); } static @@ -421,10 +549,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkVector bdcv; DeltaChunkVector tdcv; DeltaChunkVector dcv; + DeltaChunkVector tmpl; DCV_init(&bdcv, 0); DCV_init(&dcv, 0); DCV_init(&tdcv, 0); - + DCV_init(&tmpl, 200); unsigned int dsi; PyObject* ds; @@ -453,6 +582,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // parse command stream ull tbw = 0; // Amount of target bytes written + bool shared_data = dsi != 0; + bool is_first_run = dsi == 0; + assert(data < dend); while (data < dend) { @@ -481,9 +613,12 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } else if (cmd) { // TODO: Compress nodes by parsing them in advance + // NOTE: Compression only necessary for all other deltas, not + // for the first one, as we will share the data. It really depends + // What's faster DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd); + DC_set_data(dc, data, cmd, shared_data); tbw += cmd; data += cmd; } else { @@ -493,18 +628,20 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } }// END handle command opcodes assert(tbw == target_size); - + + if (!is_first_run){ + DCV_connect_with_base(&tdcv, &dcv, &tmpl); + } // swap the vector // Skip the first vector, as it is also used as top chunk vector if (bdcv.mem != tdcv.mem){ - DCV_dealloc(&bdcv); + DCV_destroy(&bdcv); } bdcv = dcv; - if (dsi == 0){ + if (is_first_run){ tdcv = dcv; } DCV_init(&dcv, 0); - loop_end: // perform cleanup @@ -524,10 +661,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - DCV_dealloc(&bdcv); + DCV_destroy(&tmpl); + DCV_destroy(&bdcv); if (dsi > 1){ // otherwise dcv equals tcl - DCV_dealloc(&dcv); + DCV_destroy(&dcv); } // Return the actual python object - its just a container @@ -535,7 +673,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); // Otherwise tdcv would be deallocated by the chunk list - DCV_dealloc(&tdcv); + DCV_destroy(&tdcv); error = 1; } else { // Plain copy, don't deallocate From 166e538f9aab8db7ab30b6e8b3be407200a7e3c1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 18:17:19 +0200 Subject: [PATCH 0084/1790] Enhanced memory handling within the delta-stream parsing method. Removed the base delta chunk vector, which was a reminder of old (python) times which are long gone --- _fun.c | 88 +++++++++++++++++++++++++++++++++++++++------------------- fun.py | 4 +-- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/_fun.c b/_fun.c index 1186c6d11..142795e6c 100644 --- a/_fun.c +++ b/_fun.c @@ -198,46 +198,62 @@ typedef struct { Py_ssize_t reserved_size; // Reserve in DeltaChunks } DeltaChunkVector; -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ + + +// Reserve enough memory to hold the given amount of delta chunks +// Return 1 on success inline -int DCV_grow(DeltaChunkVector* vec, uint num_dc) +int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) { - const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; - if (grow_by_chunks <= 0){ + if (num_dc <= vec->reserved_size){ return 1; } +#ifdef DEBUG + bool was_null = vec->mem == NULL; +#endif + if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(DeltaChunk)); + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); } if (vec->mem == NULL){ Py_FatalError("Could not allocate memory for append operation"); } - vec->reserved_size = vec->reserved_size + grow_by_chunks; + vec->reserved_size = num_dc; #ifdef DEBUG - fprintf(stderr, "Allocated %i bytes at %p, to hold up to %i chunks\n", (int)((vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)), vec->mem, (int)(vec->reserved_size + grow_by_chunks)); + const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; + if (!was_null) + format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); #endif return vec->mem != NULL; } +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +inline +int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +{ + return DCV_reserve_memory(vec, vec->reserved_size + num_dc); +} + int DCV_init(DeltaChunkVector* vec, ull initial_size) { vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; - return DCV_grow(vec, initial_size); + return DCV_grow_by(vec, initial_size); } inline @@ -309,6 +325,24 @@ void DCV_forget_members(DeltaChunkVector* vec) vec->size = 0; } +// Reset the vector so that its size will be zero, and its members will +// have been deallocated properly. +// It will keep its memory though, and hence can be filled again +inline +void DCV_reset(DeltaChunkVector* vec) +{ + if (vec->size == 0) + return; + + DeltaChunk* dc = vec->mem; + DeltaChunk* dcend = DCV_end(vec); + for(;dc < dcend; dc++){ + DC_destroy(dc); + } + + vec->size = 0; +} + // Append num-chunks to the end of the list, possibly reallocating existing ones // Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory @@ -316,7 +350,7 @@ static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) { if (vec->size + num_chunks > vec->reserved_size){ - DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size); + DCV_grow_by(vec, (vec->size + num_chunks) - vec->reserved_size); } Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; @@ -337,7 +371,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow(vec, 1); + DCV_grow_by(vec, 1); } DeltaChunk* next = vec->mem + vec->size; @@ -546,12 +580,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkVector bdcv; - DeltaChunkVector tdcv; DeltaChunkVector dcv; + DeltaChunkVector tdcv; DeltaChunkVector tmpl; - DCV_init(&bdcv, 0); - DCV_init(&dcv, 0); + DCV_init(&dcv, 100); // should be enough to keep the average text file DCV_init(&tdcv, 0); DCV_init(&tmpl, 200); @@ -578,7 +610,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // estimate number of ops - assume one third adds, half two byte (size+offset) copies const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); - DCV_grow(&dcv, approx_num_cmds); + DCV_reserve_memory(&dcv, approx_num_cmds); // parse command stream ull tbw = 0; // Amount of target bytes written @@ -632,16 +664,15 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!is_first_run){ DCV_connect_with_base(&tdcv, &dcv, &tmpl); } - // swap the vector - // Skip the first vector, as it is also used as top chunk vector - if (bdcv.mem != tdcv.mem){ - DCV_destroy(&bdcv); - } - bdcv = dcv; + if (is_first_run){ tdcv = dcv; + // wipe out dcv without destroying the members, get its own memory + DCV_init(&dcv, tdcv.size); + } else { + // destroy members, but keep memory + DCV_reset(&dcv); } - DCV_init(&dcv, 0); loop_end: // perform cleanup @@ -662,7 +693,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } DCV_destroy(&tmpl); - DCV_destroy(&bdcv); if (dsi > 1){ // otherwise dcv equals tcl DCV_destroy(&dcv); diff --git a/fun.py b/fun.py index e6262b4bc..13a3c627f 100644 --- a/fun.py +++ b/fun.py @@ -545,7 +545,6 @@ def connect_deltas(dstreams): :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" - bdcl = None # data chunk list for initial base tdcl = None # topmost dcl dcl = tdcl = TopdownDeltaChunkList() @@ -611,13 +610,12 @@ def connect_deltas(dstreams): dcl.compress() # merge the lists ! - if bdcl is not None: + if dsi > 0: if not tdcl.connect_with_next_base(dcl): break # END handle merge # prepare next base - bdcl = dcl dcl = DeltaChunkList() # END for each delta stream From 60f7768ed00ad666317c1877abe52906d6014d50 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 19:46:24 +0200 Subject: [PATCH 0085/1790] Implemented everything about the merging of the bases into the topmost delta list. Its not yet working, but at least its not crashing --- _fun.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/_fun.c b/_fun.c index 142795e6c..eb4b6b865 100644 --- a/_fun.c +++ b/_fun.c @@ -91,6 +91,9 @@ typedef unsigned int uint; typedef unsigned char uchar; typedef uchar bool; +// Constants +const ull gDVC_grow_by = 50; + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -277,10 +280,17 @@ DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) return &vec->mem[i]; } +// Return last item +inline +DeltaChunk* DCV_last(DeltaChunkVector* vec) +{ + return DCV_get(vec, vec->size-1); +} + inline ull DCV_rbound(DeltaChunkVector* vec) { - return DC_rbound(DCV_get(vec, vec->size-1)); + return DC_rbound(DCV_last(vec)); } inline @@ -371,7 +381,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, 1); + DCV_grow_by(vec, gDVC_grow_by); } DeltaChunk* next = vec->mem + vec->size; @@ -379,6 +389,33 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) return next; } +// Return delta chunk being closest to the given absolute offset +inline +DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) +{ + assert(vec->mem); + + ull lo = 0; + ull hi = vec->size; + ull mid; + DeltaChunk* dc; + + while (lo < hi) + { + mid = (lo + hi) / 2; + dc = vec->mem + mid; + if (dc->to > ofs){ + hi = mid; + } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + return dc; + } else { + lo = mid + 1; + } + } + + return DCV_last(vec); +} + // Write a slice as defined by its absolute offset in bytes and its size into the given // destination. The individual chunks written will be a deep copy of the source // data chunks @@ -387,14 +424,67 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) inline void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { + assert(DCV_lbound(src) <= ofs); + assert(DCV_rbound(src) <= ofs + size); + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + DeltaChunk* destc = DCV_append(dest); + const ull relofs = ofs - cdc->to; + DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + cdc += 1; + size -= destc->ts; + + if (size == 0){ + return; + } + } + + DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc) + { + if (cdc->ts < size) { + DC_copy_to(cdc, DCV_append(dest)); + size -= cdc->ts; + } else { + DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + size = 0; + break; + } + } + + assert(size == 0); } +// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which +// originates in to +// 'at' will be replaced by the items to insert ( special purpose ) +// 'at' will be properly destroyed, but all items will just be copied bytewise +// using memcpy. Hence from must just forget about them ! +inline +void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +{ + assert(from->size > 1); + + DCV_reserve_memory(to, to->size + from->size - 1); // -1 because we replace at + DC_destroy(at); + to->size -= 1 + from->size; + + // If we are somewhere in the middle, we have to make some space + if (DCV_last(to) != at) { + memmove((void*)at+from->size, (void*)(at+1), (size_t)(DCV_end(to) - (at+1))); + } + + // Finally copy all the items in + memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); +} + // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the // caller -static void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) { DeltaChunk* dc = tdcv->mem; @@ -413,18 +503,20 @@ void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, Delta // assert(tmpl->size); // move target bounds - DeltaChunk* cdc = tmpl->mem; - DeltaChunk* cdcend = tmpl->mem + tmpl->size; + DeltaChunk* tdc = tmpl->mem; + DeltaChunk* tdcend = tmpl->mem + tmpl->size; const ull ofs = dc->to - dc->so; - for(;cdc < cdcend; cdc++){ - cdc->to += ofs; + for(;tdc < tdcend; tdc++){ + tdc->to += ofs; } - // insert slice into our list, replacing our current chunk + // insert slice into our list if (tmpl->size == 1){ + // Its not data, so destroy is not really required, anyhow ... + DC_destroy(dc); *dc = *DCV_get(tmpl, 0); } else { - + DCV_replace_one_by_many(tmpl, tdcv, dc); } // make sure the members will not be deallocated by the list From 60b4f37a8b17c8f89b1b0ba7a0268f375469033d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 22:06:27 +0200 Subject: [PATCH 0086/1790] Improved performance of python implementation by 10 percent, just by removing function calls and object creations --- fun.py | 82 ++++++++++++---------------------------------------------- 1 file changed, 17 insertions(+), 65 deletions(-) diff --git a/fun.py b/fun.py index 13a3c627f..038b4c761 100644 --- a/fun.py +++ b/fun.py @@ -55,9 +55,6 @@ def _set_delta_rbound(d, size): :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size :return: d""" - if d.ts == size: - return - d.ts = size # NOTE: data is truncated automatically when applying the delta @@ -154,76 +151,30 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 -def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): +def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed write function. :param bbuf: base buffer containing the base of all deltas contained in this list. It will only be used if the chunk in question does not have a base chain. - :param lbound_offset: offset at which to start applying the delta, relative to - our lbound - :param size: if larger than 0, only the given amount of bytes will be applied :param write: function taking a string of bytes to write to the output""" - slen = len(dcl) - if slen == 0: - return - # END early abort - absofs = dcl.lbound() + lbound_offset - if size == 0: - size = dcl.rbound() - absofs - # END initialize size - - if lbound_offset or absofs + size != dcl.rbound(): - cdi = _closest_index(dcl, absofs) - cd = dcl[cdi] - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - delta_chunk_apply(tcd, bbuf, write) - size -= tcd.ts - cdi += 1 - # END handle first chunk - - # here we have to either apply full chunks, or smaller ones, but - # we always start at the chunks target offset - while cdi < slen and size: - cd = dcl[cdi] - if cd.ts <= size: - delta_chunk_apply(cd, bbuf, write) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - delta_chunk_apply(tcd, bbuf, write) - size -= tcd.ts - break - # END handle bytes to apply - cdi += 1 - # END handle rest - else: - for dc in dcl: - delta_chunk_apply(dc, bbuf, write) - # END for each dc - # END handle application values + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc -def delta_list_slice(dcl, absofs, size): +def delta_list_slice(dcl, absofs, size, ndcl): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. - :return: list (copy) which represents the given chunk""" - dcllbound = dcl.lbound() - absofs = max(absofs, dcllbound) - size = min(dcl.rbound() - dcllbound, size) + :return: None""" cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = list() lappend = ndcl.append if cd.to != absofs: - tcd = delta_duplicate(cd) + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) + tcd.ts = min(tcd.ts, size) lappend(tcd) size -= tcd.ts cdi += 1 @@ -233,11 +184,11 @@ def delta_list_slice(dcl, absofs, size): # are we larger than the current block cd = dcl[cdi] if cd.ts <= size: - lappend(delta_duplicate(cd)) + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) size -= cd.ts else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size lappend(tcd) size -= tcd.ts break @@ -245,7 +196,6 @@ def delta_list_slice(dcl, absofs, size): cdi += 1 # END for each chunk - return ndcl class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. @@ -272,10 +222,10 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def apply(self, bbuf, write, lbound_offset=0, size=0): + def apply(self, bbuf, write): """Only used by public clients, internally we only use the global routines for performance""" - return delta_list_apply(self, bbuf, write, lbound_offset, size) + return delta_list_apply(self, bbuf, write) def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate @@ -369,6 +319,7 @@ def connect_with_next_base(self, bdcl): nfc = 0 # number of frozen chunks dci = 0 # delta chunk index slen = len(self) # len of self + ccl = list() # temporary list while dci < slen: dc = self[dci] dci += 1 @@ -384,8 +335,9 @@ def connect_with_next_base(self, bdcl): # dont support efficient insertion ( just one at a time ), but for now # we live with it. Internally, its all just a 32/64bit pointer, and # the portions of moved memory should be smallish. Maybe we just rebuild - # ourselves in order to reduce the amount of insertions ... - ccl = delta_list_slice(bdcl, dc.so, dc.ts) + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) # move the target bounds into place to match with our chunk ofs = dc.to - dc.so From a63ee1d034a6677bc3ab408d1a16593bbb6078ca Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 23:55:45 +0200 Subject: [PATCH 0087/1790] Currently there is a weird memory bug, valgrind says it is writing one byte too much. Perhaps its because of the use of PyMem --- _fun.c | 115 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/_fun.c b/_fun.c index eb4b6b865..fc4ee1edc 100644 --- a/_fun.c +++ b/_fun.c @@ -94,6 +94,12 @@ typedef uchar bool; // Constants const ull gDVC_grow_by = 50; +#ifdef DEBUG +#define DBG_check(vec) DCV_dbg_check_integrity(vec) +#else +#define DBG_check(vec) +#endif + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -154,19 +160,20 @@ void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared } inline -ull DC_rbound(DeltaChunk* dc) +ull DC_rbound(const DeltaChunk* dc) { return dc->to + dc->ts; } // Copy all data from src to dest, the data pointer will be copied too inline -void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) +void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) { dest->to = src->to; dest->ts = src->ts; dest->so = src->so; dest->data_shared = 0; + dest->data = NULL; DC_set_data(dest, src->data, src->ts, 0); } @@ -174,7 +181,7 @@ void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) // Copy all data with the given offset and size. The source offset, as well // as the data will be truncated accordingly inline -void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) { assert(size <= src->ts); assert(src->to + ofs + size <= DC_rbound(src)); @@ -182,6 +189,7 @@ void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) dest->to = src->to + ofs; dest->ts = size; dest->so = src->so + ofs; + dest->data = NULL; if (src->data){ DC_set_data(dest, src->data + ofs, size, 0); @@ -260,13 +268,13 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) } inline -ull DCV_len(DeltaChunkVector* vec) +ull DCV_len(const DeltaChunkVector* vec) { return vec->size; } inline -ull DCV_lbound(DeltaChunkVector* vec) +ull DCV_lbound(const DeltaChunkVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -274,7 +282,7 @@ ull DCV_lbound(DeltaChunkVector* vec) // Return item at index inline -DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) +DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; @@ -282,29 +290,29 @@ DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) // Return last item inline -DeltaChunk* DCV_last(DeltaChunkVector* vec) +DeltaChunk* DCV_last(const DeltaChunkVector* vec) { return DCV_get(vec, vec->size-1); } inline -ull DCV_rbound(DeltaChunkVector* vec) +ull DCV_rbound(const DeltaChunkVector* vec) { return DC_rbound(DCV_last(vec)); } inline -int DCV_empty(DeltaChunkVector* vec) +int DCV_empty(const DeltaChunkVector* vec) { return vec->size == 0; } // Return end pointer of the vector inline -DeltaChunk* DCV_end(DeltaChunkVector* vec) +const DeltaChunk* DCV_end(const DeltaChunkVector* vec) { assert(!DCV_empty(vec)); - return &vec->mem[vec->size]; + return vec->mem + vec->size; } void DCV_destroy(DeltaChunkVector* vec) @@ -345,7 +353,7 @@ void DCV_reset(DeltaChunkVector* vec) return; DeltaChunk* dc = vec->mem; - DeltaChunk* dcend = DCV_end(vec); + const DeltaChunk* dcend = DCV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); } @@ -366,11 +374,9 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_ssize_t old_size = vec->size; vec->size += num_chunks; -#ifdef DEBUG for(;old_size < vec->size; ++old_size){ DC_init(DCV_get(vec, old_size), 0, 0, 0); } -#endif return &vec->mem[old_size]; } @@ -391,7 +397,7 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) // Return delta chunk being closest to the given absolute offset inline -DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) +DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) { assert(vec->mem); @@ -416,16 +422,43 @@ DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) return DCV_last(vec); } +// Assert the given vector has correct datachunks +void DCV_dbg_check_integrity(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + const DeltaChunk* i = vec->mem; + const DeltaChunk* end = DCV_end(vec); + + ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); + ull acc_size = 0; + for(; i < end; i++){ + acc_size += i->ts; + } + assert(acc_size == aparent_size); + + if (vec->size < 2){ + return; + } + + const DeltaChunk* endm1 = DCV_end(vec) - 1; + for(i = vec->mem; i < endm1; i++){ + const DeltaChunk* n = i+1; + assert(DC_rbound(i) == n->to); + } + +} + // Write a slice as defined by its absolute offset in bytes and its size into the given // destination. The individual chunks written will be a deep copy of the source // data chunks // TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort // of append-only memory pool would improve performance inline -void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { + //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); assert(DCV_lbound(src) <= ofs); - assert(DCV_rbound(src) <= ofs + size); + assert((ofs + size) <= DCV_rbound(src)); DeltaChunk* cdc = DCV_closest_chunk(src, ofs); @@ -442,7 +475,7 @@ void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, u } } - DeltaChunk* vecend = DCV_end(src); + const DeltaChunk* vecend = DCV_end(src); for( ;(cdc < vecend) && size; ++cdc) { if (cdc->ts < size) { @@ -464,18 +497,22 @@ void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, u // 'at' will be replaced by the items to insert ( special purpose ) // 'at' will be properly destroyed, but all items will just be copied bytewise // using memcpy. Hence from must just forget about them ! +// IMPORTANT: to must have an appropriate size already inline -void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { + fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); + assert(to->size + from->size - 1 <= to->reserved_size); - DCV_reserve_memory(to, to->size + from->size - 1); // -1 because we replace at + // -1 because we replace 'at' DC_destroy(at); - to->size -= 1 + from->size; + to->size += from->size - 1; // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - memmove((void*)at+from->size, (void*)(at+1), (size_t)(DCV_end(to) - (at+1))); + fprintf(stderr, "moving to %p from %p, num bytes = %i\n", at+from->size, at+1, (int)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)(DCV_end(to) - (at+1)) * sizeof(DeltaChunk)); } // Finally copy all the items in @@ -485,22 +522,27 @@ void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, Delta // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the // caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) { - DeltaChunk* dc = tdcv->mem; - DeltaChunk* end = tdcv->mem + tdcv->size; - assert(dc); + Py_ssize_t dci = 0; + Py_ssize_t iend = tdcv->size; + DeltaChunk* dc; - for (;dc < end; dc++) + DBG_check(tdcv); + DBG_check(bdcv); + + for (;dci < iend; dci++) { // Data chunks don't need processing + dc = DCV_get(tdcv, dci); if (dc->data){ continue; } // Copy Chunk Handling DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - // assert(tmpl->size); + DBG_check(tmpl); + assert(tmpl->size); // move target bounds DeltaChunk* tdc = tmpl->mem; @@ -516,8 +558,15 @@ void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, Delta DC_destroy(dc); *dc = *DCV_get(tmpl, 0); } else { + DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); + dc = DCV_get(tdcv, dci); DCV_replace_one_by_many(tmpl, tdcv, dc); + // Compensate for us being replaced + dci += tmpl->size-1; + iend += tmpl->size-1; } + + DBG_check(tdcv); // make sure the members will not be deallocated by the list DCV_forget_members(tmpl); @@ -679,8 +728,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DCV_init(&tdcv, 0); DCV_init(&tmpl, 200); - unsigned int dsi; - PyObject* ds; + unsigned int dsi = 0; + PyObject* ds = 0; int error = 0; for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) { @@ -706,7 +755,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // parse command stream ull tbw = 0; // Amount of target bytes written - bool shared_data = dsi != 0; + bool is_shared_data = dsi != 0; bool is_first_run = dsi == 0; assert(data < dend); @@ -742,10 +791,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // What's faster DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd, shared_data); + DC_set_data(dc, data, cmd, is_shared_data); tbw += cmd; data += cmd; - } else { + } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); goto loop_end; From 4098056f4fe101f1e50d924ab3ba40650d563b9d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 00:14:06 +0200 Subject: [PATCH 0088/1790] Fixed terrible bug, which happened due to a change of the size of the vector, but too early actually, so a memmove would use incorrect values --- _fun.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/_fun.c b/_fun.c index fc4ee1edc..3be3bccab 100644 --- a/_fun.c +++ b/_fun.c @@ -501,22 +501,24 @@ void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull inline void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { - fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); + //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); assert(to->size + from->size - 1 <= to->reserved_size); // -1 because we replace 'at' DC_destroy(at); - to->size += from->size - 1; // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - fprintf(stderr, "moving to %p from %p, num bytes = %i\n", at+from->size, at+1, (int)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - memmove((void*)(at+from->size), (void*)(at+1), (size_t)(DCV_end(to) - (at+1)) * sizeof(DeltaChunk)); + //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } - + // Finally copy all the items in memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + + // FINALLY: update size + to->size += from->size - 1; } // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost From f563a9db73a24c6353794c8e093cc604be9c7d7b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 00:49:27 +0200 Subject: [PATCH 0089/1790] Fixed integrity check function, finalized code, so far it is working, and its very efficient as well as the amount of data-copies is minimized --- _fun.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/_fun.c b/_fun.c index 3be3bccab..0b485473f 100644 --- a/_fun.c +++ b/_fun.c @@ -95,7 +95,7 @@ typedef uchar bool; const ull gDVC_grow_by = 50; #ifdef DEBUG -#define DBG_check(vec) DCV_dbg_check_integrity(vec) +#define DBG_check(vec) asser(DCV_dbg_check_integrity(vec)) #else #define DBG_check(vec) #endif @@ -165,6 +165,26 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } +// Apply +inline +void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) +{ + PyObject* buffer = 0; + if (dc->data){ + buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); + } else { + buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); + } + + if (PyTuple_SetItem(tmpargs, 0, buffer)){ + assert(0); + } + + // tuple steals reference, and will take care about the deallocation + PyObject_Call(writer, tmpargs, NULL); + +} + // Copy all data from src to dest, the data pointer will be copied too inline void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) @@ -423,9 +443,12 @@ DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) } // Assert the given vector has correct datachunks -void DCV_dbg_check_integrity(const DeltaChunkVector* vec) +// return 1 on success +int DCV_dbg_check_integrity(const DeltaChunkVector* vec) { - assert(!DCV_empty(vec)); + if(DCV_empty(vec)){ + return 0; + } const DeltaChunk* i = vec->mem; const DeltaChunk* end = DCV_end(vec); @@ -434,18 +457,22 @@ void DCV_dbg_check_integrity(const DeltaChunkVector* vec) for(; i < end; i++){ acc_size += i->ts; } - assert(acc_size == aparent_size); + if (acc_size != aparent_size) + return 0; if (vec->size < 2){ - return; + return 1; } const DeltaChunk* endm1 = DCV_end(vec) - 1; for(i = vec->mem; i < endm1; i++){ const DeltaChunk* n = i+1; - assert(DC_rbound(i) == n->to); + if (DC_rbound(i) != n->to){ + return 0; + } } + return 1; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -624,10 +651,41 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) return PyLong_FromUnsignedLongLong(DCL_rbound(self)); } +// Write using a write function, taking remaining bytes from a base buffer static -PyObject* DCL_apply(PyObject* self, PyObject* args) +PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + PyObject* pybuf = 0; + PyObject* writeproc = 0; + if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ + PyErr_BadArgument(); + return NULL; + } + + if (!PyObject_CheckReadBuffer(pybuf)){ + PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + + if (!PyCallable_Check(writeproc)){ + PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); + return NULL; + } + + const DeltaChunk* i = self->vec.mem; + const DeltaChunk* end = DCV_end(&self->vec); + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + + PyObject* tmpargs = PyTuple_New(1); + + for(; i < end; i++){ + DC_apply(i, data, writeproc, tmpargs); + } + Py_DECREF(tmpargs); Py_RETURN_NONE; } From 64992444e9a2b2936cea4d2cba235e59e703cac9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 10:46:21 +0200 Subject: [PATCH 0090/1790] optimized reallocation count, which improves speed a little bit. Previously it would easily get into the habbit of reallocating the vector just to add a single item --- _fun.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/_fun.c b/_fun.c index 0b485473f..1373a5c60 100644 --- a/_fun.c +++ b/_fun.c @@ -92,10 +92,10 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDVC_grow_by = 50; +const ull gDVC_grow_by = 100; #ifdef DEBUG -#define DBG_check(vec) asser(DCV_dbg_check_integrity(vec)) +#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) #else #define DBG_check(vec) #endif @@ -233,6 +233,8 @@ typedef struct { // Reserve enough memory to hold the given amount of delta chunks // Return 1 on success +// NOTE: added a minimum allocation to assure reallocation is not done +// just for a single additional entry. DCVs change often, and reallocs are expensive inline int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) { @@ -240,6 +242,10 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) return 1; } + if (num_dc - vec->reserved_size){ + num_dc += gDVC_grow_by; + } + #ifdef DEBUG bool was_null = vec->mem == NULL; #endif @@ -381,25 +387,6 @@ void DCV_reset(DeltaChunkVector* vec) vec->size = 0; } -// Append num-chunks to the end of the list, possibly reallocating existing ones -// Return a pointer to the first of the added items. They are already null initialized -// If num-chunks == 0, it returns the end pointer of the allocated memory -static inline -DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) -{ - if (vec->size + num_chunks > vec->reserved_size){ - DCV_grow_by(vec, (vec->size + num_chunks) - vec->reserved_size); - } - Py_FatalError("Could not allocate memory for append operation"); - Py_ssize_t old_size = vec->size; - vec->size += num_chunks; - - for(;old_size < vec->size; ++old_size){ - DC_init(DCV_get(vec, old_size), 0, 0, 0); - } - - return &vec->mem[old_size]; -} // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! @@ -838,6 +825,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ + assert(0); break; } @@ -849,6 +837,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // NOTE: Compression only necessary for all other deltas, not // for the first one, as we will share the data. It really depends // What's faster + // Compression reduces fragmentation though, which is why we do it + // in all cases. DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); DC_set_data(dc, data, cmd, is_shared_data); @@ -860,7 +850,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } }// END handle command opcodes - assert(tbw == target_size); + if (tbw != target_size){ + PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); + error = 1; + } if (!is_first_run){ DCV_connect_with_base(&tdcv, &dcv, &tmpl); From a7253b8d08ffc69bf3afdec98619044c84c64f75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 12:19:06 +0200 Subject: [PATCH 0091/1790] implemented memory compression, but got evil memory bug once again ... probably its just as stupid as previously --- _fun.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/_fun.c b/_fun.c index 1373a5c60..cf1f5b629 100644 --- a/_fun.c +++ b/_fun.c @@ -159,6 +159,16 @@ void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared } +// Make the given data our own. It is assumed to have the size stored in our instance +// and will be managed by us. +inline +void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) +{ + assert(data); + DC_deallocate_data(dc); + dc->data = data; +} + inline ull DC_rbound(const DeltaChunk* dc) { @@ -214,7 +224,6 @@ void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull siz if (src->data){ DC_set_data(dest, src->data + ofs, size, 0); } else { - dest->data = NULL; dest->data_shared = 0; } } @@ -825,6 +834,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ + // this really shouldn't happen + error = 1; assert(0); break; } @@ -834,16 +845,53 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } else if (cmd) { // TODO: Compress nodes by parsing them in advance - // NOTE: Compression only necessary for all other deltas, not - // for the first one, as we will share the data. It really depends - // What's faster // Compression reduces fragmentation though, which is why we do it // in all cases. - DeltaChunk* dc = DCV_append(&dcv); - DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd, is_shared_data); - tbw += cmd; + const uchar* add_start = data - 1; + const uchar* add_end = dend; + ull num_bytes = cmd; data += cmd; + ull num_chunks = 1; + while (data < dend){ + fprintf(stderr, "looping\n"); + const char c = *data; + if (c & 0x80){ + add_end = data; + break; + } else { + num_chunks += 1; + data += c + 1; // advance by 1 to skip add cmd + num_bytes += c; + } + } + + fprintf(stderr, "add bytes = %i\n", (int)num_bytes); + #ifdef DEBUG + assert(add_end - add_start > 0); + if (num_chunks > 1){ + fprintf(stderr, "Compression worked, got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + } + #endif + + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, num_bytes, 0); + + // gather the data, or (possibly) share single blocks + if (num_chunks > 1){ + uchar* dcdata = PyMem_Malloc(num_bytes); + while (add_start < add_end){ + const char bytes = *add_start++; + fprintf(stderr, "Copying %i bytes\n", bytes); + memcpy((void*)dcdata, (void*)add_start, bytes); + dcdata += bytes; + add_start += bytes; + } + DC_set_data_with_ownership(dc, dcdata); + } else { + DC_set_data(dc, data - cmd, cmd, is_shared_data); + } + + tbw += num_bytes; } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); From bd01f6932c12fe2b7010884d339cc4831cc7ec59 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:27:38 +0200 Subject: [PATCH 0092/1790] Fixed memory bug, it was a small tiny thing, as well as stupid. --- _fun.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/_fun.c b/_fun.c index cf1f5b629..247008682 100644 --- a/_fun.c +++ b/_fun.c @@ -844,32 +844,32 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) tbw += cp_size; } else if (cmd) { - // TODO: Compress nodes by parsing them in advance // Compression reduces fragmentation though, which is why we do it // in all cases. + // It makes the more sense the more consecutive add-chunks we have, + // its more likely in big deltas, for big binary files const uchar* add_start = data - 1; const uchar* add_end = dend; ull num_bytes = cmd; data += cmd; ull num_chunks = 1; while (data < dend){ - fprintf(stderr, "looping\n"); + //while (0){ const char c = *data; if (c & 0x80){ add_end = data; break; } else { - num_chunks += 1; - data += c + 1; // advance by 1 to skip add cmd + data += 1 + c; // advance by 1 to skip add cmd num_bytes += c; + num_chunks += 1; } } - fprintf(stderr, "add bytes = %i\n", (int)num_bytes); #ifdef DEBUG assert(add_end - add_start > 0); if (num_chunks > 1){ - fprintf(stderr, "Compression worked, got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); } #endif @@ -881,12 +881,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) uchar* dcdata = PyMem_Malloc(num_bytes); while (add_start < add_end){ const char bytes = *add_start++; - fprintf(stderr, "Copying %i bytes\n", bytes); memcpy((void*)dcdata, (void*)add_start, bytes); dcdata += bytes; add_start += bytes; } - DC_set_data_with_ownership(dc, dcdata); + DC_set_data_with_ownership(dc, dcdata-num_bytes); } else { DC_set_data(dc, data - cmd, cmd, is_shared_data); } From 5442e4aec5741d5b346ab9a7cd091976a8f5e9f4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:48:44 +0200 Subject: [PATCH 0093/1790] Put delta-apply code into separate function. Would have preferred to to have just one dynamic module, lets see whether includes are possible --- Makefile | 3 + _delta_apply.c | 903 +++++++++++++++++++++++++++++++++++++++++++++++++ _fun.c | 885 ------------------------------------------------ fun.py | 2 +- setup.py | 5 +- 5 files changed, 911 insertions(+), 887 deletions(-) create mode 100644 _delta_apply.c diff --git a/Makefile b/Makefile index 190a66b03..e65c55a6d 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ build:: $(SETUP) build $(SETUP) build_ext -i +build_ext:: + $(SETUP) build_ext -i + install:: $(SETUP) install diff --git a/_delta_apply.c b/_delta_apply.c new file mode 100644 index 000000000..40ea60c56 --- /dev/null +++ b/_delta_apply.c @@ -0,0 +1,903 @@ +#include +#include +#include +#include +#include +#include + +typedef unsigned long long ull; +typedef unsigned int uint; +typedef unsigned char uchar; +typedef uchar bool; + +// Constants +const ull gDVC_grow_by = 100; + +#ifdef DEBUG +#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) +#else +#define DBG_check(vec) +#endif + +// DELTA CHUNK +//////////////// +// Internal Delta Chunk Objects +typedef struct { + ull to; + ull ts; + ull so; + const uchar* data; + bool data_shared; +} DeltaChunk; + +inline +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) +{ + dc->to = to; + dc->ts = ts; + dc->so = so; + dc->data = NULL; + dc->data_shared = 0; +} + +inline +void DC_deallocate_data(DeltaChunk* dc) +{ + if (!dc->data_shared && dc->data){ + PyMem_Free((void*)dc->data); + } + dc->data = NULL; +} + +inline +void DC_destroy(DeltaChunk* dc) +{ + DC_deallocate_data(dc); +} + +// Store a copy of data in our instance. If shared is 1, the data will be shared, +// hence it will only be stored, but the memory will not be touched, or copied. +inline +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) +{ + DC_deallocate_data(dc); + + if (data == 0){ + dc->data = NULL; + dc->data_shared = 0; + return; + } + + dc->data_shared = shared; + if (shared){ + dc->data = data; + } else { + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy((void*)dc->data, (void*)data, dlen); + } + +} + +// Make the given data our own. It is assumed to have the size stored in our instance +// and will be managed by us. +inline +void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) +{ + assert(data); + DC_deallocate_data(dc); + dc->data = data; +} + +inline +ull DC_rbound(const DeltaChunk* dc) +{ + return dc->to + dc->ts; +} + +// Apply +inline +void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) +{ + PyObject* buffer = 0; + if (dc->data){ + buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); + } else { + buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); + } + + if (PyTuple_SetItem(tmpargs, 0, buffer)){ + assert(0); + } + + // tuple steals reference, and will take care about the deallocation + PyObject_Call(writer, tmpargs, NULL); + +} + +// Copy all data from src to dest, the data pointer will be copied too +inline +void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) +{ + dest->to = src->to; + dest->ts = src->ts; + dest->so = src->so; + dest->data_shared = 0; + dest->data = NULL; + + DC_set_data(dest, src->data, src->ts, 0); +} + +// Copy all data with the given offset and size. The source offset, as well +// as the data will be truncated accordingly +inline +void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +{ + assert(size <= src->ts); + assert(src->to + ofs + size <= DC_rbound(src)); + + dest->to = src->to + ofs; + dest->ts = size; + dest->so = src->so + ofs; + dest->data = NULL; + + if (src->data){ + DC_set_data(dest, src->data + ofs, size, 0); + } else { + dest->data_shared = 0; + } +} + + +// DELTA CHUNK VECTOR +///////////////////// + +typedef struct { + DeltaChunk* mem; // Memory + Py_ssize_t size; // Size in DeltaChunks + Py_ssize_t reserved_size; // Reserve in DeltaChunks +} DeltaChunkVector; + + + +// Reserve enough memory to hold the given amount of delta chunks +// Return 1 on success +// NOTE: added a minimum allocation to assure reallocation is not done +// just for a single additional entry. DCVs change often, and reallocs are expensive +inline +int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) +{ + if (num_dc <= vec->reserved_size){ + return 1; + } + + if (num_dc - vec->reserved_size){ + num_dc += gDVC_grow_by; + } + +#ifdef DEBUG + bool was_null = vec->mem == NULL; +#endif + + if (vec->mem == NULL){ + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); + } else { + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); + } + + if (vec->mem == NULL){ + Py_FatalError("Could not allocate memory for append operation"); + } + + vec->reserved_size = num_dc; + +#ifdef DEBUG + const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; + if (!was_null) + format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); +#endif + + return vec->mem != NULL; +} + +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +inline +int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +{ + return DCV_reserve_memory(vec, vec->reserved_size + num_dc); +} + +int DCV_init(DeltaChunkVector* vec, ull initial_size) +{ + vec->mem = NULL; + vec->size = 0; + vec->reserved_size = 0; + + return DCV_grow_by(vec, initial_size); +} + +inline +ull DCV_len(const DeltaChunkVector* vec) +{ + return vec->size; +} + +inline +ull DCV_lbound(const DeltaChunkVector* vec) +{ + assert(vec->size && vec->mem); + return vec->mem->to; +} + +// Return item at index +inline +DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) +{ + assert(i < vec->size && vec->mem); + return &vec->mem[i]; +} + +// Return last item +inline +DeltaChunk* DCV_last(const DeltaChunkVector* vec) +{ + return DCV_get(vec, vec->size-1); +} + +inline +ull DCV_rbound(const DeltaChunkVector* vec) +{ + return DC_rbound(DCV_last(vec)); +} + +inline +int DCV_empty(const DeltaChunkVector* vec) +{ + return vec->size == 0; +} + +// Return end pointer of the vector +inline +const DeltaChunk* DCV_end(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return vec->mem + vec->size; +} + +void DCV_destroy(DeltaChunkVector* vec) +{ + if (vec->mem){ +#ifdef DEBUG + fprintf(stderr, "Freeing %p\n", (void*)vec->mem); +#endif + + const DeltaChunk* end = &vec->mem[vec->size]; + DeltaChunk* i; + for(i = vec->mem; i < end; i++){ + DC_destroy(i); + } + + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +// Reset this vector so that its existing memory can be filled again. +// Memory will be kept, but not cleaned up +inline +void DCV_forget_members(DeltaChunkVector* vec) +{ + vec->size = 0; +} + +// Reset the vector so that its size will be zero, and its members will +// have been deallocated properly. +// It will keep its memory though, and hence can be filled again +inline +void DCV_reset(DeltaChunkVector* vec) +{ + if (vec->size == 0) + return; + + DeltaChunk* dc = vec->mem; + const DeltaChunk* dcend = DCV_end(vec); + for(;dc < dcend; dc++){ + DC_destroy(dc); + } + + vec->size = 0; +} + + +// Append one chunk to the end of the list, and return a pointer to it +// It will not have been initialized ! +static inline +DeltaChunk* DCV_append(DeltaChunkVector* vec) +{ + if (vec->size + 1 > vec->reserved_size){ + DCV_grow_by(vec, gDVC_grow_by); + } + + DeltaChunk* next = vec->mem + vec->size; + vec->size += 1; + return next; +} + +// Return delta chunk being closest to the given absolute offset +inline +DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) +{ + assert(vec->mem); + + ull lo = 0; + ull hi = vec->size; + ull mid; + DeltaChunk* dc; + + while (lo < hi) + { + mid = (lo + hi) / 2; + dc = vec->mem + mid; + if (dc->to > ofs){ + hi = mid; + } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + return dc; + } else { + lo = mid + 1; + } + } + + return DCV_last(vec); +} + +// Assert the given vector has correct datachunks +// return 1 on success +int DCV_dbg_check_integrity(const DeltaChunkVector* vec) +{ + if(DCV_empty(vec)){ + return 0; + } + const DeltaChunk* i = vec->mem; + const DeltaChunk* end = DCV_end(vec); + + ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); + ull acc_size = 0; + for(; i < end; i++){ + acc_size += i->ts; + } + if (acc_size != aparent_size) + return 0; + + if (vec->size < 2){ + return 1; + } + + const DeltaChunk* endm1 = DCV_end(vec) - 1; + for(i = vec->mem; i < endm1; i++){ + const DeltaChunk* n = i+1; + if (DC_rbound(i) != n->to){ + return 0; + } + } + + return 1; +} + +// Write a slice as defined by its absolute offset in bytes and its size into the given +// destination. The individual chunks written will be a deep copy of the source +// data chunks +// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort +// of append-only memory pool would improve performance +inline +void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +{ + //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); + assert(DCV_lbound(src) <= ofs); + assert((ofs + size) <= DCV_rbound(src)); + + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + DeltaChunk* destc = DCV_append(dest); + const ull relofs = ofs - cdc->to; + DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + cdc += 1; + size -= destc->ts; + + if (size == 0){ + return; + } + } + + const DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc) + { + if (cdc->ts < size) { + DC_copy_to(cdc, DCV_append(dest)); + size -= cdc->ts; + } else { + DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + size = 0; + break; + } + } + + assert(size == 0); +} + + +// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which +// originates in to +// 'at' will be replaced by the items to insert ( special purpose ) +// 'at' will be properly destroyed, but all items will just be copied bytewise +// using memcpy. Hence from must just forget about them ! +// IMPORTANT: to must have an appropriate size already +inline +void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +{ + //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); + assert(from->size > 1); + assert(to->size + from->size - 1 <= to->reserved_size); + + // -1 because we replace 'at' + DC_destroy(at); + + // If we are somewhere in the middle, we have to make some space + if (DCV_last(to) != at) { + //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); + } + + // Finally copy all the items in + memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + + // FINALLY: update size + to->size += from->size - 1; +} + +// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost +// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the +// caller +void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +{ + Py_ssize_t dci = 0; + Py_ssize_t iend = tdcv->size; + DeltaChunk* dc; + + DBG_check(tdcv); + DBG_check(bdcv); + + for (;dci < iend; dci++) + { + // Data chunks don't need processing + dc = DCV_get(tdcv, dci); + if (dc->data){ + continue; + } + + // Copy Chunk Handling + DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); + DBG_check(tmpl); + assert(tmpl->size); + + // move target bounds + DeltaChunk* tdc = tmpl->mem; + DeltaChunk* tdcend = tmpl->mem + tmpl->size; + const ull ofs = dc->to - dc->so; + for(;tdc < tdcend; tdc++){ + tdc->to += ofs; + } + + // insert slice into our list + if (tmpl->size == 1){ + // Its not data, so destroy is not really required, anyhow ... + DC_destroy(dc); + *dc = *DCV_get(tmpl, 0); + } else { + DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); + dc = DCV_get(tdcv, dci); + DCV_replace_one_by_many(tmpl, tdcv, dc); + // Compensate for us being replaced + dci += tmpl->size-1; + iend += tmpl->size-1; + } + + DBG_check(tdcv); + + // make sure the members will not be deallocated by the list + DCV_forget_members(tmpl); + } +} + +// DELTA CHUNK LIST (PYTHON) +///////////////////////////// + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunkVector vec; + +} DeltaChunkList; + + +static +int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) +{ + if(args && PySequence_Size(args) > 0){ + PyErr_SetString(PyExc_ValueError, "Too many arguments"); + return -1; + } + + DCV_init(&self->vec, 0); + return 0; +} + +static +void DCL_dealloc(DeltaChunkList* self) +{ + DCV_destroy(&(self->vec)); +} + +static +PyObject* DCL_len(DeltaChunkList* self) +{ + return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); +} + +static inline +ull DCL_rbound(DeltaChunkList* self) +{ + if (DCV_empty(&self->vec)) + return 0; + return DCV_rbound(&self->vec); +} + +static +PyObject* DCL_py_rbound(DeltaChunkList* self) +{ + return PyLong_FromUnsignedLongLong(DCL_rbound(self)); +} + +// Write using a write function, taking remaining bytes from a base buffer +static +PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) +{ + PyObject* pybuf = 0; + PyObject* writeproc = 0; + if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ + PyErr_BadArgument(); + return NULL; + } + + if (!PyObject_CheckReadBuffer(pybuf)){ + PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + + if (!PyCallable_Check(writeproc)){ + PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); + return NULL; + } + + const DeltaChunk* i = self->vec.mem; + const DeltaChunk* end = DCV_end(&self->vec); + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + + PyObject* tmpargs = PyTuple_New(1); + + for(; i < end; i++){ + DC_apply(i, data, writeproc, tmpargs); + } + + Py_DECREF(tmpargs); + Py_RETURN_NONE; +} + +static PyMethodDef DCL_methods[] = { + {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, + {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject DeltaChunkListType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "DeltaChunkList", /*tp_name*/ + sizeof(DeltaChunkList), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)DCL_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "Minimal Delta Chunk List",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DCL_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)DCL_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +// Makes a new copy of the DeltaChunkList - you have to do everything yourselve +// in C ... want C++ !! +DeltaChunkList* DCL_new_instance(void) +{ + DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); + assert(dcl); + + DCL_init(dcl, 0, 0); + assert(dcl->vec.size == 0); + assert(dcl->vec.mem == NULL); + return dcl; +} + +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; + return size; +} + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +{ + // obtain iterator + PyObject* stream_iter = 0; + if (!PyIter_Check(dstreams)){ + stream_iter = PyObject_GetIter(dstreams); + if (!stream_iter){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); + return NULL; + } + } else { + stream_iter = dstreams; + } + + DeltaChunkVector dcv; + DeltaChunkVector tdcv; + DeltaChunkVector tmpl; + DCV_init(&dcv, 100); // should be enough to keep the average text file + DCV_init(&tdcv, 0); + DCV_init(&tmpl, 200); + + unsigned int dsi = 0; + PyObject* ds = 0; + int error = 0; + for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + { + PyObject* db = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(db)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); + goto loop_end; + } + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + const uchar* dend = data + dlen; + + // read header + const ull base_size = msb_size(&data, dend); + const ull target_size = msb_size(&data, dend); + + // estimate number of ops - assume one third adds, half two byte (size+offset) copies + const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + DCV_reserve_memory(&dcv, approx_num_cmds); + + // parse command stream + ull tbw = 0; // Amount of target bytes written + bool is_shared_data = dsi != 0; + bool is_first_run = dsi == 0; + + assert(data < dend); + while (data < dend) + { + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + const unsigned long rbound = cp_off + cp_size; + if (rbound < cp_size || + rbound > base_size){ + // this really shouldn't happen + error = 1; + assert(0); + break; + } + + DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); + tbw += cp_size; + + } else if (cmd) { + // Compression reduces fragmentation though, which is why we do it + // in all cases. + // It makes the more sense the more consecutive add-chunks we have, + // its more likely in big deltas, for big binary files + const uchar* add_start = data - 1; + const uchar* add_end = dend; + ull num_bytes = cmd; + data += cmd; + ull num_chunks = 1; + while (data < dend){ + //while (0){ + const char c = *data; + if (c & 0x80){ + add_end = data; + break; + } else { + data += 1 + c; // advance by 1 to skip add cmd + num_bytes += c; + num_chunks += 1; + } + } + + #ifdef DEBUG + assert(add_end - add_start > 0); + if (num_chunks > 1){ + fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + } + #endif + + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, num_bytes, 0); + + // gather the data, or (possibly) share single blocks + if (num_chunks > 1){ + uchar* dcdata = PyMem_Malloc(num_bytes); + while (add_start < add_end){ + const char bytes = *add_start++; + memcpy((void*)dcdata, (void*)add_start, bytes); + dcdata += bytes; + add_start += bytes; + } + DC_set_data_with_ownership(dc, dcdata-num_bytes); + } else { + DC_set_data(dc, data - cmd, cmd, is_shared_data); + } + + tbw += num_bytes; + } else { + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + goto loop_end; + } + }// END handle command opcodes + if (tbw != target_size){ + PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); + error = 1; + } + + if (!is_first_run){ + DCV_connect_with_base(&tdcv, &dcv, &tmpl); + } + + if (is_first_run){ + tdcv = dcv; + // wipe out dcv without destroying the members, get its own memory + DCV_init(&dcv, tdcv.size); + } else { + // destroy members, but keep memory + DCV_reset(&dcv); + } + +loop_end: + // perform cleanup + Py_DECREF(ds); + Py_DECREF(db); + + if (error){ + break; + } + }// END for each stream object + + if (dsi == 0 && ! error){ + PyErr_SetString(PyExc_ValueError, "No streams provided"); + } + + if (stream_iter != dstreams){ + Py_DECREF(stream_iter); + } + + DCV_destroy(&tmpl); + if (dsi > 1){ + // otherwise dcv equals tcl + DCV_destroy(&dcv); + } + + // Return the actual python object - its just a container + DeltaChunkList* dcl = DCL_new_instance(); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + // Otherwise tdcv would be deallocated by the chunk list + DCV_destroy(&tdcv); + error = 1; + } else { + // Plain copy, don't deallocate + dcl->vec = tdcv; + } + + if (error){ + // Will dealloc tdcv + Py_XDECREF(dcl); + return NULL; + } + + return (PyObject*)dcl; +} + +static PyMethodDef py_fun[] = { + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, + { NULL, NULL, 0, NULL } +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC init_delta_apply(void) +{ + PyObject *m; + + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + + m = Py_InitModule3("_delta_apply", py_fun, NULL); + if (m == NULL) + return; + + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); +} diff --git a/_fun.c b/_fun.c index 247008682..1881bfb05 100644 --- a/_fun.c +++ b/_fun.c @@ -1,9 +1,4 @@ #include -#include -#include -#include -#include -#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -86,883 +81,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) } -typedef unsigned long long ull; -typedef unsigned int uint; -typedef unsigned char uchar; -typedef uchar bool; - -// Constants -const ull gDVC_grow_by = 100; - -#ifdef DEBUG -#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) -#else -#define DBG_check(vec) -#endif - -// DELTA CHUNK -//////////////// -// Internal Delta Chunk Objects -typedef struct { - ull to; - ull ts; - ull so; - const uchar* data; - bool data_shared; -} DeltaChunk; - -inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) -{ - dc->to = to; - dc->ts = ts; - dc->so = so; - dc->data = NULL; - dc->data_shared = 0; -} - -inline -void DC_deallocate_data(DeltaChunk* dc) -{ - if (!dc->data_shared && dc->data){ - PyMem_Free((void*)dc->data); - } - dc->data = NULL; -} - -inline -void DC_destroy(DeltaChunk* dc) -{ - DC_deallocate_data(dc); -} - -// Store a copy of data in our instance. If shared is 1, the data will be shared, -// hence it will only be stored, but the memory will not be touched, or copied. -inline -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) -{ - DC_deallocate_data(dc); - - if (data == 0){ - dc->data = NULL; - dc->data_shared = 0; - return; - } - - dc->data_shared = shared; - if (shared){ - dc->data = data; - } else { - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy((void*)dc->data, (void*)data, dlen); - } - -} - -// Make the given data our own. It is assumed to have the size stored in our instance -// and will be managed by us. -inline -void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) -{ - assert(data); - DC_deallocate_data(dc); - dc->data = data; -} - -inline -ull DC_rbound(const DeltaChunk* dc) -{ - return dc->to + dc->ts; -} - -// Apply -inline -void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) -{ - PyObject* buffer = 0; - if (dc->data){ - buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); - } else { - buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); - } - - if (PyTuple_SetItem(tmpargs, 0, buffer)){ - assert(0); - } - - // tuple steals reference, and will take care about the deallocation - PyObject_Call(writer, tmpargs, NULL); - -} - -// Copy all data from src to dest, the data pointer will be copied too -inline -void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) -{ - dest->to = src->to; - dest->ts = src->ts; - dest->so = src->so; - dest->data_shared = 0; - dest->data = NULL; - - DC_set_data(dest, src->data, src->ts, 0); -} - -// Copy all data with the given offset and size. The source offset, as well -// as the data will be truncated accordingly -inline -void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) -{ - assert(size <= src->ts); - assert(src->to + ofs + size <= DC_rbound(src)); - - dest->to = src->to + ofs; - dest->ts = size; - dest->so = src->so + ofs; - dest->data = NULL; - - if (src->data){ - DC_set_data(dest, src->data + ofs, size, 0); - } else { - dest->data_shared = 0; - } -} - - -// DELTA CHUNK VECTOR -///////////////////// - -typedef struct { - DeltaChunk* mem; // Memory - Py_ssize_t size; // Size in DeltaChunks - Py_ssize_t reserved_size; // Reserve in DeltaChunks -} DeltaChunkVector; - - - -// Reserve enough memory to hold the given amount of delta chunks -// Return 1 on success -// NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DCVs change often, and reallocs are expensive -inline -int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) -{ - if (num_dc <= vec->reserved_size){ - return 1; - } - - if (num_dc - vec->reserved_size){ - num_dc += gDVC_grow_by; - } - -#ifdef DEBUG - bool was_null = vec->mem == NULL; -#endif - - if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); - } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); - } - - if (vec->mem == NULL){ - Py_FatalError("Could not allocate memory for append operation"); - } - - vec->reserved_size = num_dc; - -#ifdef DEBUG - const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; - if (!was_null) - format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); -#endif - - return vec->mem != NULL; -} - -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ -inline -int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) -{ - return DCV_reserve_memory(vec, vec->reserved_size + num_dc); -} - -int DCV_init(DeltaChunkVector* vec, ull initial_size) -{ - vec->mem = NULL; - vec->size = 0; - vec->reserved_size = 0; - - return DCV_grow_by(vec, initial_size); -} - -inline -ull DCV_len(const DeltaChunkVector* vec) -{ - return vec->size; -} - -inline -ull DCV_lbound(const DeltaChunkVector* vec) -{ - assert(vec->size && vec->mem); - return vec->mem->to; -} - -// Return item at index -inline -DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) -{ - assert(i < vec->size && vec->mem); - return &vec->mem[i]; -} - -// Return last item -inline -DeltaChunk* DCV_last(const DeltaChunkVector* vec) -{ - return DCV_get(vec, vec->size-1); -} - -inline -ull DCV_rbound(const DeltaChunkVector* vec) -{ - return DC_rbound(DCV_last(vec)); -} - -inline -int DCV_empty(const DeltaChunkVector* vec) -{ - return vec->size == 0; -} - -// Return end pointer of the vector -inline -const DeltaChunk* DCV_end(const DeltaChunkVector* vec) -{ - assert(!DCV_empty(vec)); - return vec->mem + vec->size; -} - -void DCV_destroy(DeltaChunkVector* vec) -{ - if (vec->mem){ -#ifdef DEBUG - fprintf(stderr, "Freeing %p\n", (void*)vec->mem); -#endif - - const DeltaChunk* end = &vec->mem[vec->size]; - DeltaChunk* i; - for(i = vec->mem; i < end; i++){ - DC_destroy(i); - } - - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - -// Reset this vector so that its existing memory can be filled again. -// Memory will be kept, but not cleaned up -inline -void DCV_forget_members(DeltaChunkVector* vec) -{ - vec->size = 0; -} - -// Reset the vector so that its size will be zero, and its members will -// have been deallocated properly. -// It will keep its memory though, and hence can be filled again -inline -void DCV_reset(DeltaChunkVector* vec) -{ - if (vec->size == 0) - return; - - DeltaChunk* dc = vec->mem; - const DeltaChunk* dcend = DCV_end(vec); - for(;dc < dcend; dc++){ - DC_destroy(dc); - } - - vec->size = 0; -} - - -// Append one chunk to the end of the list, and return a pointer to it -// It will not have been initialized ! -static inline -DeltaChunk* DCV_append(DeltaChunkVector* vec) -{ - if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDVC_grow_by); - } - - DeltaChunk* next = vec->mem + vec->size; - vec->size += 1; - return next; -} - -// Return delta chunk being closest to the given absolute offset -inline -DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) -{ - assert(vec->mem); - - ull lo = 0; - ull hi = vec->size; - ull mid; - DeltaChunk* dc; - - while (lo < hi) - { - mid = (lo + hi) / 2; - dc = vec->mem + mid; - if (dc->to > ofs){ - hi = mid; - } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { - return dc; - } else { - lo = mid + 1; - } - } - - return DCV_last(vec); -} - -// Assert the given vector has correct datachunks -// return 1 on success -int DCV_dbg_check_integrity(const DeltaChunkVector* vec) -{ - if(DCV_empty(vec)){ - return 0; - } - const DeltaChunk* i = vec->mem; - const DeltaChunk* end = DCV_end(vec); - - ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); - ull acc_size = 0; - for(; i < end; i++){ - acc_size += i->ts; - } - if (acc_size != aparent_size) - return 0; - - if (vec->size < 2){ - return 1; - } - - const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = vec->mem; i < endm1; i++){ - const DeltaChunk* n = i+1; - if (DC_rbound(i) != n->to){ - return 0; - } - } - - return 1; -} - -// Write a slice as defined by its absolute offset in bytes and its size into the given -// destination. The individual chunks written will be a deep copy of the source -// data chunks -// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort -// of append-only memory pool would improve performance -inline -void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) -{ - //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); - assert(DCV_lbound(src) <= ofs); - assert((ofs + size) <= DCV_rbound(src)); - - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); - - // partial overlap - if (cdc->to != ofs) { - DeltaChunk* destc = DCV_append(dest); - const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); - cdc += 1; - size -= destc->ts; - - if (size == 0){ - return; - } - } - - const DeltaChunk* vecend = DCV_end(src); - for( ;(cdc < vecend) && size; ++cdc) - { - if (cdc->ts < size) { - DC_copy_to(cdc, DCV_append(dest)); - size -= cdc->ts; - } else { - DC_offset_copy_to(cdc, DCV_append(dest), 0, size); - size = 0; - break; - } - } - - assert(size == 0); -} - - -// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which -// originates in to -// 'at' will be replaced by the items to insert ( special purpose ) -// 'at' will be properly destroyed, but all items will just be copied bytewise -// using memcpy. Hence from must just forget about them ! -// IMPORTANT: to must have an appropriate size already -inline -void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) -{ - //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); - assert(from->size > 1); - assert(to->size + from->size - 1 <= to->reserved_size); - - // -1 because we replace 'at' - DC_destroy(at); - - // If we are somewhere in the middle, we have to make some space - if (DCV_last(to) != at) { - //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); - memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - } - - // Finally copy all the items in - memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); - - // FINALLY: update size - to->size += from->size - 1; -} - -// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the -// caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) -{ - Py_ssize_t dci = 0; - Py_ssize_t iend = tdcv->size; - DeltaChunk* dc; - - DBG_check(tdcv); - DBG_check(bdcv); - - for (;dci < iend; dci++) - { - // Data chunks don't need processing - dc = DCV_get(tdcv, dci); - if (dc->data){ - continue; - } - - // Copy Chunk Handling - DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - DBG_check(tmpl); - assert(tmpl->size); - - // move target bounds - DeltaChunk* tdc = tmpl->mem; - DeltaChunk* tdcend = tmpl->mem + tmpl->size; - const ull ofs = dc->to - dc->so; - for(;tdc < tdcend; tdc++){ - tdc->to += ofs; - } - - // insert slice into our list - if (tmpl->size == 1){ - // Its not data, so destroy is not really required, anyhow ... - DC_destroy(dc); - *dc = *DCV_get(tmpl, 0); - } else { - DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); - dc = DCV_get(tdcv, dci); - DCV_replace_one_by_many(tmpl, tdcv, dc); - // Compensate for us being replaced - dci += tmpl->size-1; - iend += tmpl->size-1; - } - - DBG_check(tdcv); - - // make sure the members will not be deallocated by the list - DCV_forget_members(tmpl); - } -} - -// DELTA CHUNK LIST (PYTHON) -///////////////////////////// - -typedef struct { - PyObject_HEAD - // ----------- - DeltaChunkVector vec; - -} DeltaChunkList; - - -static -int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) -{ - if(args && PySequence_Size(args) > 0){ - PyErr_SetString(PyExc_ValueError, "Too many arguments"); - return -1; - } - - DCV_init(&self->vec, 0); - return 0; -} - -static -void DCL_dealloc(DeltaChunkList* self) -{ - DCV_destroy(&(self->vec)); -} - -static -PyObject* DCL_len(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); -} - -static inline -ull DCL_rbound(DeltaChunkList* self) -{ - if (DCV_empty(&self->vec)) - return 0; - return DCV_rbound(&self->vec); -} - -static -PyObject* DCL_py_rbound(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DCL_rbound(self)); -} - -// Write using a write function, taking remaining bytes from a base buffer -static -PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) -{ - PyObject* pybuf = 0; - PyObject* writeproc = 0; - if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ - PyErr_BadArgument(); - return NULL; - } - - if (!PyObject_CheckReadBuffer(pybuf)){ - PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - - if (!PyCallable_Check(writeproc)){ - PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); - return NULL; - } - - const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DCV_end(&self->vec); - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); - - PyObject* tmpargs = PyTuple_New(1); - - for(; i < end; i++){ - DC_apply(i, data, writeproc, tmpargs); - } - - Py_DECREF(tmpargs); - Py_RETURN_NONE; -} - -static PyMethodDef DCL_methods[] = { - {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, - {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject DeltaChunkListType = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "DeltaChunkList", /*tp_name*/ - sizeof(DeltaChunkList), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)DCL_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "Minimal Delta Chunk List",/* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - DCL_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -// Makes a new copy of the DeltaChunkList - you have to do everything yourselve -// in C ... want C++ !! -DeltaChunkList* DCL_new_instance(void) -{ - DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); - assert(dcl); - - DCL_init(dcl, 0, 0); - assert(dcl->vec.size == 0); - assert(dcl->vec.mem == NULL); - return dcl; -} - -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) -{ - // obtain iterator - PyObject* stream_iter = 0; - if (!PyIter_Check(dstreams)){ - stream_iter = PyObject_GetIter(dstreams); - if (!stream_iter){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); - return NULL; - } - } else { - stream_iter = dstreams; - } - - DeltaChunkVector dcv; - DeltaChunkVector tdcv; - DeltaChunkVector tmpl; - DCV_init(&dcv, 100); // should be enough to keep the average text file - DCV_init(&tdcv, 0); - DCV_init(&tmpl, 200); - - unsigned int dsi = 0; - PyObject* ds = 0; - int error = 0; - for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) - { - PyObject* db = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(db)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); - goto loop_end; - } - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* dend = data + dlen; - - // read header - const ull base_size = msb_size(&data, dend); - const ull target_size = msb_size(&data, dend); - - // estimate number of ops - assume one third adds, half two byte (size+offset) copies - const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); - DCV_reserve_memory(&dcv, approx_num_cmds); - - // parse command stream - ull tbw = 0; // Amount of target bytes written - bool is_shared_data = dsi != 0; - bool is_first_run = dsi == 0; - - assert(data < dend); - while (data < dend) - { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - const unsigned long rbound = cp_off + cp_size; - if (rbound < cp_size || - rbound > base_size){ - // this really shouldn't happen - error = 1; - assert(0); - break; - } - - DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); - tbw += cp_size; - - } else if (cmd) { - // Compression reduces fragmentation though, which is why we do it - // in all cases. - // It makes the more sense the more consecutive add-chunks we have, - // its more likely in big deltas, for big binary files - const uchar* add_start = data - 1; - const uchar* add_end = dend; - ull num_bytes = cmd; - data += cmd; - ull num_chunks = 1; - while (data < dend){ - //while (0){ - const char c = *data; - if (c & 0x80){ - add_end = data; - break; - } else { - data += 1 + c; // advance by 1 to skip add cmd - num_bytes += c; - num_chunks += 1; - } - } - - #ifdef DEBUG - assert(add_end - add_start > 0); - if (num_chunks > 1){ - fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); - } - #endif - - DeltaChunk* dc = DCV_append(&dcv); - DC_init(dc, tbw, num_bytes, 0); - - // gather the data, or (possibly) share single blocks - if (num_chunks > 1){ - uchar* dcdata = PyMem_Malloc(num_bytes); - while (add_start < add_end){ - const char bytes = *add_start++; - memcpy((void*)dcdata, (void*)add_start, bytes); - dcdata += bytes; - add_start += bytes; - } - DC_set_data_with_ownership(dc, dcdata-num_bytes); - } else { - DC_set_data(dc, data - cmd, cmd, is_shared_data); - } - - tbw += num_bytes; - } else { - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - goto loop_end; - } - }// END handle command opcodes - if (tbw != target_size){ - PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); - error = 1; - } - - if (!is_first_run){ - DCV_connect_with_base(&tdcv, &dcv, &tmpl); - } - - if (is_first_run){ - tdcv = dcv; - // wipe out dcv without destroying the members, get its own memory - DCV_init(&dcv, tdcv.size); - } else { - // destroy members, but keep memory - DCV_reset(&dcv); - } - -loop_end: - // perform cleanup - Py_DECREF(ds); - Py_DECREF(db); - - if (error){ - break; - } - }// END for each stream object - - if (dsi == 0 && ! error){ - PyErr_SetString(PyExc_ValueError, "No streams provided"); - } - - if (stream_iter != dstreams){ - Py_DECREF(stream_iter); - } - - DCV_destroy(&tmpl); - if (dsi > 1){ - // otherwise dcv equals tcl - DCV_destroy(&dcv); - } - - // Return the actual python object - its just a container - DeltaChunkList* dcl = DCL_new_instance(); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdcv would be deallocated by the chunk list - DCV_destroy(&tdcv); - error = 1; - } else { - // Plain copy, don't deallocate - dcl->vec = tdcv; - } - - if (error){ - // Will dealloc tdcv - Py_XDECREF(dcl); - return NULL; - } - - return (PyObject*)dcl; -} - static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; @@ -973,13 +93,8 @@ PyMODINIT_FUNC init_fun(void) { PyObject *m; - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - m = Py_InitModule3("_fun", py_fun, NULL); if (m == NULL) return; - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "Noddy", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index 038b4c761..e17460bf6 100644 --- a/fun.py +++ b/fun.py @@ -654,6 +654,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: # raise ImportError; # DEBUG - from _fun import connect_deltas + from _delta_apply import connect_deltas except ImportError: pass diff --git a/setup.py b/setup.py index 265156df2..7146ea833 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,10 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._fun', ['_fun.c'])], + ext_modules=[ + Extension('gitdb._fun', ['_fun.c']), + Extension('gitdb._delta_apply', ['_delta_apply.c']) + ], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 9c5672e1b0cf56f1cf5151d60acb35d610e904a8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:55:22 +0200 Subject: [PATCH 0094/1790] Now building a single module called _perf which contains all the performance enhancements, which increases loadtimes, less is more --- _delta_apply.c | 22 ---------------------- _fun.c | 12 +++++++++--- fun.py | 2 +- pack.py | 2 +- setup.py | 5 +---- 5 files changed, 12 insertions(+), 31 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 40ea60c56..d81e65d15 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -879,25 +879,3 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) return (PyObject*)dcl; } -static PyMethodDef py_fun[] = { - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, - { NULL, NULL, 0, NULL } -}; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC init_delta_apply(void) -{ - PyObject *m; - - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - - m = Py_InitModule3("_delta_apply", py_fun, NULL); - if (m == NULL) - return; - - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); -} diff --git a/_fun.c b/_fun.c index 1881bfb05..386068577 100644 --- a/_fun.c +++ b/_fun.c @@ -1,4 +1,5 @@ #include +#include "_delta_apply.c" static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -80,21 +81,26 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) Py_RETURN_NONE; } - static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ #define PyMODINIT_FUNC void #endif -PyMODINIT_FUNC init_fun(void) +PyMODINIT_FUNC init_perf(void) { PyObject *m; - m = Py_InitModule3("_fun", py_fun, NULL); + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + + m = Py_InitModule3("_perf", py_fun, NULL); if (m == NULL) return; + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index e17460bf6..0b14f82ac 100644 --- a/fun.py +++ b/fun.py @@ -654,6 +654,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: # raise ImportError; # DEBUG - from _delta_apply import connect_deltas + from _perf import connect_deltas except ImportError: pass diff --git a/pack.py b/pack.py index 91323fcce..79ffcc217 100644 --- a/pack.py +++ b/pack.py @@ -24,7 +24,7 @@ ) try: - from _fun import PackIndexFile_sha_to_index + from _perf import PackIndexFile_sha_to_index except ImportError: pass # END try c module diff --git a/setup.py b/setup.py index 7146ea833..7e50e0fe7 100755 --- a/setup.py +++ b/setup.py @@ -77,10 +77,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[ - Extension('gitdb._fun', ['_fun.c']), - Extension('gitdb._delta_apply', ['_delta_apply.c']) - ], + ext_modules=[Extension('gitdb._perf', ['_fun.c'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From fa03f746746df99763deb45943988d68fb450b4b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 15:51:04 +0200 Subject: [PATCH 0095/1790] Reverse Delta Application was a nice experiment, as it has one major flaw: Currently it integrates chunks from its base into the topmost delta chunk list, which causes plenty of mem-move operations. Plenty means, many many many, and its getting worse the more deltas you have of course. The algorithm was supposed to reduce the amount of memory activity, but failed at this point, making it worse than before. Probably it would just be fastest to implement the previous python algorithm, which swaps two buffers, in c --- _delta_apply.c | 11 +++++++---- stream.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index d81e65d15..e667a2eda 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -398,7 +398,6 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) inline void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { - //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); assert(DCV_lbound(src) <= ofs); assert((ofs + size) <= DCV_rbound(src)); @@ -443,7 +442,6 @@ void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull inline void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { - //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); assert(to->size + from->size - 1 <= to->reserved_size); @@ -452,7 +450,6 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } @@ -725,7 +722,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const ull target_size = msb_size(&data, dend); // estimate number of ops - assume one third adds, half two byte (size+offset) copies - const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + // Assume good compression for the adds + const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); DCV_reserve_memory(&dcv, approx_num_cmds); // parse command stream @@ -825,6 +823,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DCV_connect_with_base(&tdcv, &dcv, &tmpl); } + #ifdef DEBUG + fprintf(stderr, "tdcv->size = %i, tdcv->reserved_size = %i\n", (int)tdcv.size, (int)tdcv.reserved_size); + fprintf(stderr, "dcv->size = %i, dcv->reserved_size = %i\n", (int)dcv.size, (int)dcv.reserved_size); + #endif + if (is_first_run){ tdcv = dcv; // wipe out dcv without destroying the members, get its own memory diff --git a/stream.py b/stream.py index af4591f85..8f9f9c386 100644 --- a/stream.py +++ b/stream.py @@ -337,6 +337,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. + # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) # call len directly, as the (optional) c version doesn't implement the sequence From ff9c83a3fea26653d725686a692d0100efb63382 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 16:28:07 +0200 Subject: [PATCH 0096/1790] Disabled new implementation in favor of the old one - all that's needed is a c implementation of apply delta data --- _delta_apply.c | 4 ++++ stream.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index e667a2eda..51f55e892 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -450,6 +450,10 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { + // IMPORTANT: This memmove kills the performance in case of large deltas + // Causing everything to slow down enormously. Its logical, as the memory + // gets shifted each time we insert nodes, for each chunk, for ever smaller + // chunks depending on the deltas memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } diff --git a/stream.py b/stream.py index 8f9f9c386..2c24426c6 100644 --- a/stream.py +++ b/stream.py @@ -325,7 +325,7 @@ def __init__(self, stream_list): self._dstreams = tuple(stream_list[:-1]) self._br = 0 - def _set_cache_(self, attr): + def _set_cache_too_slow(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python, every function call costs @@ -360,7 +360,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_brute_(self, attr): + def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From 95d820239d5444d59ae67f50de715f1574c7213b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 17:38:53 +0200 Subject: [PATCH 0097/1790] apply_delta now has a C implementation, which is only 25 percent faster for small files ( the overhead of all the rest is very high ), but 4 times faster for large files, where the enormous call overhead coming in with python really starts to show --- _delta_apply.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ _fun.c | 3 ++- stream.py | 17 ++++++------ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 51f55e892..dfb0a94c3 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -886,3 +886,73 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) return (PyObject*)dcl; } + +// Write using a write function, taking remaining bytes from a base buffer +// replaces the corresponding method in python +static +PyObject* apply_delta(PyObject* self, PyObject* args) +{ + PyObject* pybbuf = 0; + PyObject* pydbuf = 0; + PyObject* pytbuf = 0; + if (!PyArg_ParseTuple(args, "OOO", &pybbuf, &pydbuf, &pytbuf)){ + PyErr_BadArgument(); + return NULL; + } + + PyObject* objects[] = { pybbuf, pydbuf, pytbuf }; + assert(sizeof(objects) / sizeof(PyObject*) == 3); + + uint i; + for(i = 0; i < 3; i++){ + if (!PyObject_CheckReadBuffer(objects[i])){ + PyErr_SetString(PyExc_ValueError, "Argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + } + + Py_ssize_t lbbuf; Py_ssize_t ldbuf; Py_ssize_t ltbuf; + const uchar* bbuf; const uchar* dbuf; + uchar* tbuf; + PyObject_AsReadBuffer(pybbuf, (const void**)(&bbuf), &lbbuf); + PyObject_AsReadBuffer(pydbuf, (const void**)(&dbuf), &ldbuf); + + if (PyObject_AsWriteBuffer(pytbuf, (void**)(&tbuf), <buf)){ + PyErr_SetString(PyExc_ValueError, "Argument 3 must be a writable buffer"); + return NULL; + } + + const uchar* data = dbuf; + const uchar* dend = dbuf + ldbuf; + + while (data < dend) + { + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + memcpy(tbuf, bbuf + cp_off, cp_size); + tbuf += cp_size; + + } else if (cmd) { + memcpy(tbuf, data, cmd); + tbuf += cmd; + data += cmd; + } else { + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + return NULL; + } + }// END handle command opcodes + + Py_RETURN_NONE; +} diff --git a/_fun.c b/_fun.c index 386068577..befee4ec4 100644 --- a/_fun.c +++ b/_fun.c @@ -83,7 +83,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "See python implementation" }, + { "apply_delta", (PyCFunction)apply_delta, METH_VARARGS, "See python implementation" }, { NULL, NULL, 0, NULL } }; diff --git a/stream.py b/stream.py index 2c24426c6..cedb70cf3 100644 --- a/stream.py +++ b/stream.py @@ -22,6 +22,11 @@ zlib ) +try: + from _perf import apply_delta as c_apply_delta +except ImportError: + pass + __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') @@ -413,7 +418,10 @@ def _set_cache_(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf); + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### # finally, swap out source and target buffers. The target is now the @@ -430,13 +438,6 @@ def _set_cache_(self, attr): self._mm_target = bbuf self._size = final_target_size - # TODO: Once that works, figure out the ordering of the opcodes. If they - # are always in-order/sequential, an alternate implementation could - # use stream access only. Of course this would mean we would read - # all deltas in advance, analyse the opcode ranges to determine a final - # concatenated opcode list which indicates what to copy from which delta - # to which position. This preprocessing would allow true streaming - def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: From f3284c1edf3670dca0cf7fb11ad50a36fe53e782 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 19:21:48 +0200 Subject: [PATCH 0098/1790] Integrated new algorithm into the stream class, it will now be chosen depending on the context to figure out which one to use. For some reason, the c version that is slow for big files really rocks when its about small files. Its better than the respective c implementation of the normal delta apply --- stream.py | 23 ++++++++++++++++++++++- test/performance/test_pack.py | 33 +++++++++++---------------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/stream.py b/stream.py index cedb70cf3..f5e05e114 100644 --- a/stream.py +++ b/stream.py @@ -22,8 +22,10 @@ zlib ) +has_perf_mod = False try: from _perf import apply_delta as c_apply_delta + has_perf_mod = True except ImportError: pass @@ -330,7 +332,7 @@ def __init__(self, stream_list): self._dstreams = tuple(stream_list[:-1]) self._br = 0 - def _set_cache_too_slow(self, attr): + def _set_cache_too_slow_without_c(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python, every function call costs @@ -366,6 +368,15 @@ def _set_cache_too_slow(self, attr): self._mm_target.seek(0) def _set_cache_(self, attr): + """Determine which version to use depending on the configuration of the deltas + :note: we are only called if we have the performance module""" + # otherwise it depends on the amount of memory to shift around + if len(self._dstreams) > 1 and self._bstream.size < 150000: + return self._set_cache_too_slow_without_c(attr) + else: + return self._set_cache_brute_(attr) + + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() @@ -438,6 +449,13 @@ def _set_cache_(self, attr): self._mm_target = bbuf self._size = final_target_size + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + + #} END configuration + def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: @@ -654,4 +672,7 @@ def close(self): def write(self, data): return len(data) + #} END W streams + + diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index af468b0be..f9169ffb0 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -25,29 +25,18 @@ def test_pack_random_access(self): # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info - access_times = list() - for rand in range(2): - if rand: - random.shuffle(sha_list) - # END shuffle shas - st = time() - for sha in sha_list: - pdb_pack_info(sha) - # END for each sha to look up - elapsed = time() - st - access_times.append(elapsed) - - # discard cache - del(pdb._entities) - pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs (random=%i) in %f s ( %f shas/s )" % (ns, len(pdb.entities()), rand, elapsed, ns / elapsed) - # END for each random mode - elapsed_order, elapsed_rand = access_times - - # well, its never really sequencial regarding the memory patterns, but it - # shows how well the prioriy cache performs - print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100)) + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + # discard cache + del(pdb._entities) + pdb.entities() + print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + # END for each random mode # query info and streams only max_items = 10000 # can wait longer when testing memory From f59c58d9478ddc49964b5bcd0711d06a957e4680 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 21:46:05 +0200 Subject: [PATCH 0099/1790] Added new stream type which will request its size from its stream. This triggers full decompression, currently, but allows work to be delayed even further.If people want a stream, they usually read it anyway. Then it doesn't matter which attriubute they query first --- base.py | 17 +++++++++++++++++ pack.py | 8 ++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/base.py b/base.py index 938a09242..d0bcc0866 100644 --- a/base.py +++ b/base.py @@ -135,6 +135,23 @@ def read(self, size=-1): @property def stream(self): return self[3] + + #} END stream reader interface + + +class ODeltaStream(OStream): + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + #} END stream reader interface diff --git a/pack.py b/pack.py index 79ffcc217..30da52c63 100644 --- a/pack.py +++ b/pack.py @@ -34,6 +34,7 @@ OStream, OPackInfo, OPackStream, + ODeltaStream, ODeltaPackInfo, ODeltaPackStream, ) @@ -584,14 +585,9 @@ def _object(self, sha, as_stream, index=-1): # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) - buf = streams[0].read(512) - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - - streams[0].stream.seek(0) # assure it can be read by the delta reader dstream = DeltaApplyReader.new(streams) - return OStream(sha, dstream.type, target_size, dstream) + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) From dcdd0fd9aa8aea6989cbf1530e73c6c86fc95761 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 19:17:22 +0200 Subject: [PATCH 0100/1790] Added initial version of a document to show possible ways to stream delta data most efficiently --- doc/source/algorithm.rst | 101 +++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 1 + stream.py | 3 ++ 3 files changed, 105 insertions(+) create mode 100644 doc/source/algorithm.rst diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst new file mode 100644 index 000000000..d1e4a9ba1 --- /dev/null +++ b/doc/source/algorithm.rst @@ -0,0 +1,101 @@ +######################## +Discussion of Algorithms +######################## + +************ +Introduction +************ +As you know, the pure-python object database support for GitPython is provided by the GitDB project. It is meant to be my backup plan to ensure that the DataVault (http://www.youtube.com/user/ByronBates99?feature=mhum#p/c/2A5C6EF5BDA8DB5C ) can handle reading huge files, especially those which were consolidated into packs. A nearly fully packed state is anticipated for the data-vaults repository, and reading these packs efficiently is an essential task. + +This document informs you about my findings in the struggle to improve the way packs are read to reduce memory usage required to handle huge files. It will try to conclude where future development could go to assure big delta-packed files can be read without the need of 8GB+ RAM. + +GitDB's main feature is the use of streams, hence the amount of memory used to read a database object is minimized, at the cost of some additional processing overhead to keep track of the stream state. This works great for legacy objects, which are essentially a zip-compressed byte-stream. + +Streaming data from delta-packed objects is far more difficult though, and only technically possible within certain limits, and at relatively high processing costs. My first observation was that there doesn't appear to be 'the one and only' algorithm which is generally superior. They all have their pros and cons, but fortunately this allows the implementation to choose the one most suited based on the amount of delta streams, as well as the size of the base, which allows an early and cheap estimate of the target size of the final data. + +********************************** +Traditional Delta-Apply-Algorithms +********************************** + +The brute-force CGit delta-apply algorithm +========================================== +CGit employs a simple and relatively brute-force algorithm, which resolves all delta streams recursively. When the recursion reaches the base level of the deltas, it will be decompressed into a buffer, then the first delta gets decompressed into a second buffer. From that, the target size of the delta can be extracted, to allocated a third buffer to hold the result of the operation, which consists of reading the delta stream byte-wise, to apply the operations in order, as described by single-byte opcodes. During recursion, each target buffer of the preceding delta-apply operation is used as base buffer for the next delta-apply operation, until the last delta was applied, leaving the final target buffer as result. + +About Delta-Opcodes +------------------- +There are only two kinds of opcodes, 'add-bytes' and 'copy-from-base'. One 'add-bytes' opcode can encode up to 7 bit of additional bytes to be copied from the delta stream into the target buffer. +A 'copy-from-base' opcode encodes a 32 bit offset into the base buffer, as well as the amount of bytes to be copied, which are up to 2^24 bytes. We may conclude that delta-bases may not be larger than 2^32+2^24 bytes in the current, extensible, implementation. +When generating the delta, git prefers copy operations over add operations, as they are much more efficient. Usually, the most recent, or biggest version of a file is used as base, whereas older and smaller versions of the file are expressed by copying only portions of the newest file. As it is not efficiently possible to represent all changes that way, add-bytes operations fill the gap where needed. All this explains why git can only add 128 bytes with one opcode, as it tries to minimize their use. +This implies that recent file history can usually be extracted faster than old history, which may involve many more deltas. + +Performance considerations +-------------------------- +The performance bottleneck of this algorithm appear to be the throughput of your RAM, as both opcodes will just trigger memcpy operations from one memory location to another, times the amount of deltas to apply. This in fact is very fast, even for big files above 100 MB. +Memory allocation could become an issue as you need the base buffer, the target buffer as well as the decompressed delta stream in memory at the same time. The continuous allocation and deallocation of possibly big buffers may support memory fragmentation. Whether it really kicks in, especially on 64 bit machines, is unproven though. +Nonetheless, the cgit implementation is currently the fastest one. + +The brute-force GitDB algorithm +=============================== +Despite of working essentially the same way as the CGit brute-force algorithm, GitDB minimizes the amount of allocations to 2 + num of deltas. The amount memory allocated peaks whiles the deltas are applied, as the base and target buffer, as well as the decompressed stream, are held in memory. +To achieve this, GitDB retrieves all delta-streams in advance, and peaks into their header information to determine the maximum size of the target buffer, just by reading 512 bytes of the compressed stream. If there is more than one delta to apply, the base buffer is set large enough to hold the biggest target buffer required by the delta streams. +Now it is possible to iterate all deltas, oldest first, newest last, and apply them using the buffers. At the end of each iteration, the buffers are swapped. + +Performance Results +------------------- +The performance test is performed on an aggressively packed repository with the history of cgit. 5000 sha's are extracted and read one after another. The delta-chains have a length of 0 to about 35. The pure-python implementation can stream the data of all objects (totaling 62,2 MiB) with an average rate of 8.1 MiB/s, which equals about 654 streams/s. +There are two bottlenecks: The major is the collection of the delta streams, which involves plenty of pack-lookup. This lookup is expensive in python, and is overly expensive. Its not overly critical though, as it only limits the amount of streams per second, not the actual data rate when applying the deltas. +Applying the deltas happens to be the second bottleneck, if the files to be processed get bigger. The more opcodes have to be processed, the more python slow function calls will dominate the result. As an example, it takes nearly 8 seconds to unpack a 125 MB file, where cgit only takes 2.4 s. + +To eliminate a few performance concerns, some key routines were rewritten in C. This changes the numbers of this particular test significantly, but not drastically, as the major bottleneck (delta collection) is still in place. Another performance reduction is due to the fact that plenty of other code related to the deltas is still pure-python. +Now all 5000 objects can be read at a rate of 11.1 MiB /s, or 892 streams/s. Fortunately, unpacking a big object is now done in 2.5s, which is just a tad slower than cgit, but with possibly less memory fragmentation. + + +************************************** +Paving the way towards delta streaming +************************************** + +GitDB's reverse delta aggregation algorithm +=========================================== +The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. + +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**), which are kept in vectors. Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. + +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream. + +The merging works by following a set of rules: + * Merge into the top-level delta from the youngest ancestor delta to the oldest one + * When merging one ADS, iterate from the first to the last chunk in TDS, then: + + * skip all add-bytes DCs. If bytes are added, these will always overwrite any operation coming from any ADS at the same offset. + * copy-from-base DCs will copy a slice of the respective portion of the ADS ( as defined by their base offset ) and use it to replace the original chunk. This acts as a 'virtual' copy-from-base operation. + + * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. + +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. + +The memory consumption during the TDS processing are the uncompressed delta-bytes, the parsed DS, as well as the TDS. Afterwards one requires an allocated base buffer, the target buffer, as well as the TDS. +It is clearly visible that the current implementation does not at all reduce memory consumption, but the opposite is true as the TDS can be large for large files. + +Performance Results +------------------- +The benchmarking context was the same as for the brute-force GitDB algorithm. This implementation is far more complex than the said brute-force implementation, which clearly reflects in the numbers. It's pure-python throughput is at only 1.1 MiB/s, which equals 89 streams/s. +The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. + +To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. +The throughput reaches 15.6 MiB/s, which equals 1267 streams/s, which makes it 14 times faster than the pure python version, and amazingly even 1.4 times faster than the brute-force C implementation. + +*TODO* +All this comes at a relatively high memory consumption, and heavily degrading performance with raising file sizes. A 125 MB file took 8 seconds to unpack for instance. The reason for this is the current implementation's brute-force algorithm to insert ADS slices into the TDS, which triggers an enormous amount of memmove operations of large portions of overlapping memory portions. + +Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. + + +Future work +=========== +* Analyse TDS to determine which sections from base buffer need to be copied. Read these in order of lowest to highest offset from base stream, and copy them into a smaller memory map. Relink source offsets to point to the new location in the new buffer. +* recompress TDS into bytestream to minimize the memory footprint when streaming, allocate the stream into a memory map. + +With that in place, streaming becomes trivial. Memory consumption + + diff --git a/doc/source/index.rst b/doc/source/index.rst index d414cb22f..5223e6bd9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: intro tutorial api + algorithm changes Indices and tables diff --git a/stream.py b/stream.py index f5e05e114..38c86dae7 100644 --- a/stream.py +++ b/stream.py @@ -379,6 +379,9 @@ def _set_cache_(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: From 6656c66f7edcf746132ce52d9ed714283140eaa0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 20:39:20 +0200 Subject: [PATCH 0101/1790] Implemented simple pre-pass to count offsets to help calculate where each chunk is going to be. This way, memmove becomes memcpy, and only one grow is required --- _delta_apply.c | 189 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 129 insertions(+), 60 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index dfb0a94c3..76d482b0d 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -11,7 +11,7 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDVC_grow_by = 100; +const ull gDCV_grow_by = 100; #ifdef DEBUG #define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) @@ -170,8 +170,8 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) return 1; } - if (num_dc - vec->reserved_size){ - num_dc += gDVC_grow_by; + if (num_dc - vec->reserved_size < 10){ + num_dc += gDCV_grow_by; } #ifdef DEBUG @@ -255,6 +255,12 @@ ull DCV_rbound(const DeltaChunkVector* vec) return DC_rbound(DCV_last(vec)); } +inline +ull DCV_size(const DeltaChunkVector* vec) +{ + return DCV_rbound(vec) - DCV_lbound(vec); +} + inline int DCV_empty(const DeltaChunkVector* vec) { @@ -269,6 +275,14 @@ const DeltaChunk* DCV_end(const DeltaChunkVector* vec) return vec->mem + vec->size; } +// return first item in vector +inline +DeltaChunk* DCV_first(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return vec->mem; +} + void DCV_destroy(DeltaChunkVector* vec) { if (vec->mem){ @@ -306,7 +320,7 @@ void DCV_reset(DeltaChunkVector* vec) if (vec->size == 0) return; - DeltaChunk* dc = vec->mem; + DeltaChunk* dc = DCV_first(vec); const DeltaChunk* dcend = DCV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); @@ -322,7 +336,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDVC_grow_by); + DCV_grow_by(vec, gDCV_grow_by); } DeltaChunk* next = vec->mem + vec->size; @@ -364,7 +378,7 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) if(DCV_empty(vec)){ return 0; } - const DeltaChunk* i = vec->mem; + const DeltaChunk* i = DCV_first(vec); const DeltaChunk* end = DCV_end(vec); ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); @@ -380,7 +394,7 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) } const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = vec->mem; i < endm1; i++){ + for(i = DCV_first(vec); i < endm1; i++){ const DeltaChunk* n = i+1; if (DC_rbound(i) != n->to){ return 0; @@ -390,46 +404,82 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) return 1; } +// Return the amount of chunks a slice at the given spot would have +inline +uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) +{ + uint num_dc = 0; + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + const ull relofs = ofs - cdc->to; + size -= cdc->ts - relofs < size ? cdc->ts - relofs : size; + num_dc += 1; + cdc += 1; + + if (size == 0){ + return num_dc; + } + } + + const DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc){ + num_dc += 1; + if (cdc->ts < size) { + size -= cdc->ts; + } else { + size = 0; + break; + } + } + + return num_dc; +} + // Write a slice as defined by its absolute offset in bytes and its size into the given -// destination. The individual chunks written will be a deep copy of the source +// destination memory. The individual chunks written will be a deep copy of the source // data chunks -// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort -// of append-only memory pool would improve performance +// Return: number of chunks in the slice inline -void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, ull size) { assert(DCV_lbound(src) <= ofs); assert((ofs + size) <= DCV_rbound(src)); DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + uint num_chunks = 0; // partial overlap if (cdc->to != ofs) { - DeltaChunk* destc = DCV_append(dest); const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); cdc += 1; - size -= destc->ts; + size -= dest->ts; + dest += 1; + num_chunks += 1; if (size == 0){ - return; + return num_chunks; } } const DeltaChunk* vecend = DCV_end(src); for( ;(cdc < vecend) && size; ++cdc) { + num_chunks += 1; if (cdc->ts < size) { - DC_copy_to(cdc, DCV_append(dest)); + DC_copy_to(cdc, dest++); size -= cdc->ts; } else { - DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + DC_offset_copy_to(cdc, dest++, 0, size); size = 0; break; } } assert(size == 0); + return num_chunks; } @@ -458,64 +508,85 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, } // Finally copy all the items in - memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + memcpy((void*) at, (void*)DCV_first(from), from->size*sizeof(DeltaChunk)); // FINALLY: update size to->size += from->size - 1; } // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the -// caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +// delta to apply. +bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) { - Py_ssize_t dci = 0; - Py_ssize_t iend = tdcv->size; - DeltaChunk* dc; - DBG_check(tdcv); DBG_check(bdcv); - for (;dci < iend; dci++) + uint* offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + if (!offset_array){ + return 0; + } + + fprintf(stderr, "old size = %i\n", (int)tdcv->size); + uint* pofs = offset_array; + uint num_addchunks = 0; + + DeltaChunk* dc = DCV_first(tdcv); + const DeltaChunk* dcend = DCV_end(tdcv); + const ull oldsize = DCV_size(tdcv); + + // OFFSET RUN + for (;dc < dcend; dc++, pofs++) { // Data chunks don't need processing - dc = DCV_get(tdcv, dci); + *pofs = num_addchunks; if (dc->data){ continue; } - // Copy Chunk Handling - DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - DBG_check(tmpl); - assert(tmpl->size); - - // move target bounds - DeltaChunk* tdc = tmpl->mem; - DeltaChunk* tdcend = tmpl->mem + tmpl->size; - const ull ofs = dc->to - dc->so; - for(;tdc < tdcend; tdc++){ - tdc->to += ofs; + // offset the next chunk by the amount of chunks in the slice + // - 1, because we replace our own chunk + num_addchunks += DCV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + } + + // reserve enough memory to hold all the new chunks + // reinit pointers, array could have been reallocated + DCV_reserve_memory(tdcv, tdcv->size + num_addchunks); + dc = DCV_last(tdcv); + dcend = DCV_first(tdcv) - 1; + + // now, that we have our pointers with the old size + tdcv->size += num_addchunks; + + // Insert slices, from the end to the beginning, which allows memcpy + // to be used, with a little help of the offset array + for (pofs -= 1; dc > dcend; dc--, pofs-- ) + { + // Data chunks don't need processing + const uint ofs = *pofs; + if (dc->data){ + // TODO: peak the preceeding chunks to figure out whether they are + // all just moved by ofs. In that case, they can move as a whole! + // just copy the chunk according to its offset + if (ofs){ + memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); + } + continue; } - // insert slice into our list - if (tmpl->size == 1){ - // Its not data, so destroy is not really required, anyhow ... - DC_destroy(dc); - *dc = *DCV_get(tmpl, 0); - } else { - DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); - dc = DCV_get(tdcv, dci); - DCV_replace_one_by_many(tmpl, tdcv, dc); - // Compensate for us being replaced - dci += tmpl->size-1; - iend += tmpl->size-1; + // Copy Chunks, and move their target offset into place + DeltaChunk* tdc = dc + ofs; + DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); + const ull relofs = dc->to - dc->so; + for(;tdc < tdcend; tdc++){ + tdc->to += relofs; } - - DBG_check(tdcv); - - // make sure the members will not be deallocated by the list - DCV_forget_members(tmpl); } + + fprintf(stderr, "NEW size = %i\n", (int)tdcv->size); + DBG_check(tdcv); + assert(DCV_size(tdcv) == oldsize); + PyMem_Free(offset_array); + return 1; } // DELTA CHUNK LIST (PYTHON) @@ -699,10 +770,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkVector dcv; DeltaChunkVector tdcv; - DeltaChunkVector tmpl; DCV_init(&dcv, 100); // should be enough to keep the average text file DCV_init(&tdcv, 0); - DCV_init(&tmpl, 200); unsigned int dsi = 0; PyObject* ds = 0; @@ -725,7 +794,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const ull base_size = msb_size(&data, dend); const ull target_size = msb_size(&data, dend); - // estimate number of ops - assume one third adds, half two byte (size+offset) copies // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); DCV_reserve_memory(&dcv, approx_num_cmds); @@ -824,7 +892,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } if (!is_first_run){ - DCV_connect_with_base(&tdcv, &dcv, &tmpl); + if (!DCV_connect_with_base(&tdcv, &dcv)){ + error = 1; + } } #ifdef DEBUG @@ -859,7 +929,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - DCV_destroy(&tmpl); if (dsi > 1){ // otherwise dcv equals tcl DCV_destroy(&dcv); From c03a46bea58d9b108cb314f9a1f0c422c05bb3bf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 21:07:21 +0200 Subject: [PATCH 0102/1790] Fixed tiny little bug that would cause our own chunk to be overridden before we make one last computation with its unaltered values --- _delta_apply.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 76d482b0d..4cdb654f2 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -456,7 +456,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); cdc += 1; size -= dest->ts; - dest += 1; + dest += 1; // must be here, we are reading the size ! num_chunks += 1; if (size == 0){ @@ -472,7 +472,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u DC_copy_to(cdc, dest++); size -= cdc->ts; } else { - DC_offset_copy_to(cdc, dest++, 0, size); + DC_offset_copy_to(cdc, dest, 0, size); size = 0; break; } @@ -526,7 +526,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) return 0; } - fprintf(stderr, "old size = %i\n", (int)tdcv->size); uint* pofs = offset_array; uint num_addchunks = 0; @@ -574,17 +573,19 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) } // Copy Chunks, and move their target offset into place + // As we could override dc when slicing, we get the data here + const ull relofs = dc->to - dc->so; + DeltaChunk* tdc = dc + ofs; DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); - const ull relofs = dc->to - dc->so; for(;tdc < tdcend; tdc++){ tdc->to += relofs; } } - fprintf(stderr, "NEW size = %i\n", (int)tdcv->size); DBG_check(tdcv); assert(DCV_size(tdcv) == oldsize); + PyMem_Free(offset_array); return 1; } From 65c9abfde0eae317c6c6dcf91918258e5e6ca33c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 21:38:48 +0200 Subject: [PATCH 0103/1790] removed some debug code --- _delta_apply.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 4cdb654f2..8cfb3f2e8 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -521,7 +521,7 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) DBG_check(tdcv); DBG_check(bdcv); - uint* offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; } @@ -531,7 +531,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) DeltaChunk* dc = DCV_first(tdcv); const DeltaChunk* dcend = DCV_end(tdcv); - const ull oldsize = DCV_size(tdcv); // OFFSET RUN for (;dc < dcend; dc++, pofs++) @@ -563,9 +562,10 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // Data chunks don't need processing const uint ofs = *pofs; if (dc->data){ - // TODO: peak the preceeding chunks to figure out whether they are + // NOTE: could peek the preceeding chunks to figure out whether they are // all just moved by ofs. In that case, they can move as a whole! - // just copy the chunk according to its offset + // tests showed that this is very rare though, even in huge deltas, so its + // not worth the extra effort if (ofs){ memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); } @@ -584,7 +584,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) } DBG_check(tdcv); - assert(DCV_size(tdcv) == oldsize); PyMem_Free(offset_array); return 1; From 78665b13ff4125f4ce3e5311d040c027bdc92a9a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 22:59:19 +0200 Subject: [PATCH 0104/1790] Updated draft with latest data, finished it by defining future ways to improve the algorithm --- doc/source/algorithm.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index d1e4a9ba1..55207b67b 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -83,19 +83,17 @@ The benchmarking context was the same as for the brute-force GitDB algorithm. Th The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. -The throughput reaches 15.6 MiB/s, which equals 1267 streams/s, which makes it 14 times faster than the pure python version, and amazingly even 1.4 times faster than the brute-force C implementation. +The throughput reaches 16.7 MiB/s, which equals 1344 streams/s, which makes it 15 times faster than the pure python version, and amazingly even 1.5 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. -*TODO* -All this comes at a relatively high memory consumption, and heavily degrading performance with raising file sizes. A 125 MB file took 8 seconds to unpack for instance. The reason for this is the current implementation's brute-force algorithm to insert ADS slices into the TDS, which triggers an enormous amount of memmove operations of large portions of overlapping memory portions. +All this comes at a relatively high memory consumption.Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. -Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. +A 125 MB file took 3.1 seconds to unpack for instance, which is only 33% slower than the c implementation of the brute-force algorithm. Future work =========== -* Analyse TDS to determine which sections from base buffer need to be copied. Read these in order of lowest to highest offset from base stream, and copy them into a smaller memory map. Relink source offsets to point to the new location in the new buffer. -* recompress TDS into bytestream to minimize the memory footprint when streaming, allocate the stream into a memory map. - -With that in place, streaming becomes trivial. Memory consumption +The current implementation of the reverse delta aggregation algorithm is already working well and fast, but leaves room for improvement in the realm of its memory consumption. One way to considerably reduce it would be to index the delta stream to determine bounds, instead of parsing it into a separate data structure +Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. From ffafb2e998220344f4674b25b6760254b2ec453e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2010 16:04:17 +0200 Subject: [PATCH 0105/1790] First adjustment to prepare the algorithm to work on deltastreams directly, without an intermediate conversion into far-too-large DeltaChunks. The new style just uses a single index, using one-fourth of the memory --- _delta_apply.c | 329 ++++++++++++++----------------------------------- 1 file changed, 93 insertions(+), 236 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 8cfb3f2e8..9768b5264 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -11,82 +11,38 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDCV_grow_by = 100; +const ull gDIV_grow_by = 100; + + +// DELTA INFO +///////////// +typedef struct { + uint dso; // delta stream offset + uint to; // target offset (cache) +} DeltaInfo; -#ifdef DEBUG -#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) -#else -#define DBG_check(vec) -#endif // DELTA CHUNK //////////////// // Internal Delta Chunk Objects +// They are just used to keep information parsed from a stream +// The data pointer is always shared typedef struct { ull to; ull ts; ull so; const uchar* data; - bool data_shared; } DeltaChunk; inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) { dc->to = to; dc->ts = ts; dc->so = so; dc->data = NULL; - dc->data_shared = 0; -} - -inline -void DC_deallocate_data(DeltaChunk* dc) -{ - if (!dc->data_shared && dc->data){ - PyMem_Free((void*)dc->data); - } - dc->data = NULL; } -inline -void DC_destroy(DeltaChunk* dc) -{ - DC_deallocate_data(dc); -} - -// Store a copy of data in our instance. If shared is 1, the data will be shared, -// hence it will only be stored, but the memory will not be touched, or copied. -inline -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) -{ - DC_deallocate_data(dc); - - if (data == 0){ - dc->data = NULL; - dc->data_shared = 0; - return; - } - - dc->data_shared = shared; - if (shared){ - dc->data = data; - } else { - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy((void*)dc->data, (void*)data, dlen); - } - -} - -// Make the given data our own. It is assumed to have the size stored in our instance -// and will be managed by us. -inline -void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) -{ - assert(data); - DC_deallocate_data(dc); - dc->data = data; -} inline ull DC_rbound(const DeltaChunk* dc) @@ -94,7 +50,8 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } -// Apply +// Apply +// TODO: remove, just left it for reference inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) { @@ -114,48 +71,16 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec } -// Copy all data from src to dest, the data pointer will be copied too -inline -void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) -{ - dest->to = src->to; - dest->ts = src->ts; - dest->so = src->so; - dest->data_shared = 0; - dest->data = NULL; - - DC_set_data(dest, src->data, src->ts, 0); -} - -// Copy all data with the given offset and size. The source offset, as well -// as the data will be truncated accordingly -inline -void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) -{ - assert(size <= src->ts); - assert(src->to + ofs + size <= DC_rbound(src)); - - dest->to = src->to + ofs; - dest->ts = size; - dest->so = src->so + ofs; - dest->data = NULL; - - if (src->data){ - DC_set_data(dest, src->data + ofs, size, 0); - } else { - dest->data_shared = 0; - } -} - // DELTA CHUNK VECTOR ///////////////////// typedef struct { - DeltaChunk* mem; // Memory - Py_ssize_t size; // Size in DeltaChunks - Py_ssize_t reserved_size; // Reserve in DeltaChunks -} DeltaChunkVector; + DeltaInfo* mem; // Memory + const uchar* dstream; // pointer to delta stream we index + Py_ssize_t size; // Size in DeltaInfos + Py_ssize_t reserved_size; // Reserve in DeltaInfos +} DeltaInfoVector; @@ -164,14 +89,14 @@ typedef struct { // NOTE: added a minimum allocation to assure reallocation is not done // just for a single additional entry. DCVs change often, and reallocs are expensive inline -int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) +int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) { if (num_dc <= vec->reserved_size){ return 1; } if (num_dc - vec->reserved_size < 10){ - num_dc += gDCV_grow_by; + num_dc += gDIV_grow_by; } #ifdef DEBUG @@ -179,9 +104,9 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) #endif if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaInfo)); } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaInfo)); } if (vec->mem == NULL){ @@ -194,7 +119,7 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; if (!was_null) format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaInfo)), vec->mem, (int)vec->reserved_size); #endif return vec->mem != NULL; @@ -207,28 +132,28 @@ large enough. Return 1 on success, 0 on failure */ inline -int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) { - return DCV_reserve_memory(vec, vec->reserved_size + num_dc); + return DIV_reserve_memory(vec, vec->reserved_size + num_dc); } -int DCV_init(DeltaChunkVector* vec, ull initial_size) +int DIV_init(DeltaInfoVector* vec, ull initial_size) { vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; - return DCV_grow_by(vec, initial_size); + return DIV_grow_by(vec, initial_size); } inline -ull DCV_len(const DeltaChunkVector* vec) +ull DIV_len(const DeltaInfoVector* vec) { return vec->size; } inline -ull DCV_lbound(const DeltaChunkVector* vec) +ull DIV_lbound(const DeltaInfoVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -236,7 +161,7 @@ ull DCV_lbound(const DeltaChunkVector* vec) // Return item at index inline -DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) +DeltaInfo* DIV_get(const DeltaInfoVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; @@ -244,54 +169,54 @@ DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) // Return last item inline -DeltaChunk* DCV_last(const DeltaChunkVector* vec) +DeltaInfo* DIV_last(const DeltaInfoVector* vec) { - return DCV_get(vec, vec->size-1); + return DIV_get(vec, vec->size-1); } inline -ull DCV_rbound(const DeltaChunkVector* vec) +ull DIV_rbound(const DeltaInfoVector* vec) { - return DC_rbound(DCV_last(vec)); + return DC_rbound(DIV_last(vec)); } inline -ull DCV_size(const DeltaChunkVector* vec) +ull DIV_size(const DeltaInfoVector* vec) { - return DCV_rbound(vec) - DCV_lbound(vec); + return DIV_rbound(vec) - DIV_lbound(vec); } inline -int DCV_empty(const DeltaChunkVector* vec) +int DIV_empty(const DeltaInfoVector* vec) { return vec->size == 0; } // Return end pointer of the vector inline -const DeltaChunk* DCV_end(const DeltaChunkVector* vec) +const DeltaInfo* DIV_end(const DeltaInfoVector* vec) { - assert(!DCV_empty(vec)); + assert(!DIV_empty(vec)); return vec->mem + vec->size; } // return first item in vector inline -DeltaChunk* DCV_first(const DeltaChunkVector* vec) +DeltaInfo* DIV_first(const DeltaInfoVector* vec) { - assert(!DCV_empty(vec)); + assert(!DIV_empty(vec)); return vec->mem; } -void DCV_destroy(DeltaChunkVector* vec) +void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG fprintf(stderr, "Freeing %p\n", (void*)vec->mem); #endif - const DeltaChunk* end = &vec->mem[vec->size]; - DeltaChunk* i; + const DeltaInfo* end = &vec->mem[vec->size]; + DeltaInfo* i; for(i = vec->mem; i < end; i++){ DC_destroy(i); } @@ -306,7 +231,7 @@ void DCV_destroy(DeltaChunkVector* vec) // Reset this vector so that its existing memory can be filled again. // Memory will be kept, but not cleaned up inline -void DCV_forget_members(DeltaChunkVector* vec) +void DIV_forget_members(DeltaInfoVector* vec) { vec->size = 0; } @@ -315,13 +240,13 @@ void DCV_forget_members(DeltaChunkVector* vec) // have been deallocated properly. // It will keep its memory though, and hence can be filled again inline -void DCV_reset(DeltaChunkVector* vec) +void DIV_reset(DeltaInfoVector* vec) { if (vec->size == 0) return; - DeltaChunk* dc = DCV_first(vec); - const DeltaChunk* dcend = DCV_end(vec); + DeltaInfo* dc = DIV_first(vec); + const DeltaInfo* dcend = DIV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); } @@ -333,27 +258,27 @@ void DCV_reset(DeltaChunkVector* vec) // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! static inline -DeltaChunk* DCV_append(DeltaChunkVector* vec) +DeltaInfo* DIV_append(DeltaInfoVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDCV_grow_by); + DIV_grow_by(vec, gDIV_grow_by); } - DeltaChunk* next = vec->mem + vec->size; + DeltaInfo* next = vec->mem + vec->size; vec->size += 1; return next; } // Return delta chunk being closest to the given absolute offset inline -DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) +DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) { assert(vec->mem); ull lo = 0; ull hi = vec->size; ull mid; - DeltaChunk* dc; + DeltaInfo* dc; while (lo < hi) { @@ -368,48 +293,16 @@ DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) } } - return DCV_last(vec); + return DIV_last(vec); } -// Assert the given vector has correct datachunks -// return 1 on success -int DCV_dbg_check_integrity(const DeltaChunkVector* vec) -{ - if(DCV_empty(vec)){ - return 0; - } - const DeltaChunk* i = DCV_first(vec); - const DeltaChunk* end = DCV_end(vec); - - ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); - ull acc_size = 0; - for(; i < end; i++){ - acc_size += i->ts; - } - if (acc_size != aparent_size) - return 0; - - if (vec->size < 2){ - return 1; - } - - const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = DCV_first(vec); i < endm1; i++){ - const DeltaChunk* n = i+1; - if (DC_rbound(i) != n->to){ - return 0; - } - } - - return 1; -} // Return the amount of chunks a slice at the given spot would have inline -uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) +uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) { uint num_dc = 0; - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + DeltaInfo* cdc = DIV_closest_chunk(src, ofs); // partial overlap if (cdc->to != ofs) { @@ -423,7 +316,7 @@ uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) } } - const DeltaChunk* vecend = DCV_end(src); + const DeltaInfo* vecend = DIV_end(src); for( ;(cdc < vecend) && size; ++cdc){ num_dc += 1; if (cdc->ts < size) { @@ -442,12 +335,12 @@ uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) // data chunks // Return: number of chunks in the slice inline -uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, ull size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) { - assert(DCV_lbound(src) <= ofs); - assert((ofs + size) <= DCV_rbound(src)); + assert(DIV_lbound(src) <= ofs); + assert((ofs + size) <= DIV_rbound(src)); - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + DeltaInfo* cdc = DIV_closest_chunk(src, ofs); uint num_chunks = 0; // partial overlap @@ -464,7 +357,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u } } - const DeltaChunk* vecend = DCV_end(src); + const DeltaInfo* vecend = DIV_end(src); for( ;(cdc < vecend) && size; ++cdc) { num_chunks += 1; @@ -483,44 +376,10 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u } -// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which -// originates in to -// 'at' will be replaced by the items to insert ( special purpose ) -// 'at' will be properly destroyed, but all items will just be copied bytewise -// using memcpy. Hence from must just forget about them ! -// IMPORTANT: to must have an appropriate size already -inline -void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) -{ - assert(from->size > 1); - assert(to->size + from->size - 1 <= to->reserved_size); - - // -1 because we replace 'at' - DC_destroy(at); - - // If we are somewhere in the middle, we have to make some space - if (DCV_last(to) != at) { - // IMPORTANT: This memmove kills the performance in case of large deltas - // Causing everything to slow down enormously. Its logical, as the memory - // gets shifted each time we insert nodes, for each chunk, for ever smaller - // chunks depending on the deltas - memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - } - - // Finally copy all the items in - memcpy((void*) at, (void*)DCV_first(from), from->size*sizeof(DeltaChunk)); - - // FINALLY: update size - to->size += from->size - 1; -} - // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. -bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) +bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) { - DBG_check(tdcv); - DBG_check(bdcv); - uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; @@ -529,8 +388,8 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) uint* pofs = offset_array; uint num_addchunks = 0; - DeltaChunk* dc = DCV_first(tdcv); - const DeltaChunk* dcend = DCV_end(tdcv); + DeltaInfo* dc = DIV_first(tdcv); + const DeltaInfo* dcend = DIV_end(tdcv); // OFFSET RUN for (;dc < dcend; dc++, pofs++) @@ -543,14 +402,14 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // offset the next chunk by the amount of chunks in the slice // - 1, because we replace our own chunk - num_addchunks += DCV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + num_addchunks += DIV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; } // reserve enough memory to hold all the new chunks // reinit pointers, array could have been reallocated - DCV_reserve_memory(tdcv, tdcv->size + num_addchunks); - dc = DCV_last(tdcv); - dcend = DCV_first(tdcv) - 1; + DIV_reserve_memory(tdcv, tdcv->size + num_addchunks); + dc = DIV_last(tdcv); + dcend = DIV_first(tdcv) - 1; // now, that we have our pointers with the old size tdcv->size += num_addchunks; @@ -567,7 +426,7 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort if (ofs){ - memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); + memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaInfo)); } continue; } @@ -576,26 +435,24 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // As we could override dc when slicing, we get the data here const ull relofs = dc->to - dc->so; - DeltaChunk* tdc = dc + ofs; - DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); + DeltaInfo* tdc = dc + ofs; + DeltaInfo* tdcend = tdc + DIV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); for(;tdc < tdcend; tdc++){ tdc->to += relofs; } } - DBG_check(tdcv); - PyMem_Free(offset_array); return 1; } // DELTA CHUNK LIST (PYTHON) ///////////////////////////// - +// Internally, it has nothing to do with a ChunkList anymore though typedef struct { PyObject_HEAD // ----------- - DeltaChunkVector vec; + DeltaInfoVector vec; } DeltaChunkList; @@ -608,28 +465,28 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - DCV_init(&self->vec, 0); + DIV_init(&self->vec, 0); return 0; } static void DCL_dealloc(DeltaChunkList* self) { - DCV_destroy(&(self->vec)); + DIV_destroy(&(self->vec)); } static PyObject* DCL_len(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); + return PyLong_FromUnsignedLongLong(DIV_len(&self->vec)); } static inline ull DCL_rbound(DeltaChunkList* self) { - if (DCV_empty(&self->vec)) + if (DIV_empty(&self->vec)) return 0; - return DCV_rbound(&self->vec); + return DIV_rbound(&self->vec); } static @@ -660,7 +517,7 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) } const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DCV_end(&self->vec); + const DeltaChunk* end = DIV_end(&self->vec); const uchar* data; Py_ssize_t dlen; @@ -768,10 +625,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkVector dcv; - DeltaChunkVector tdcv; - DCV_init(&dcv, 100); // should be enough to keep the average text file - DCV_init(&tdcv, 0); + DeltaInfoVector dcv; + DeltaInfoVector tdcv; + DIV_init(&dcv, 100); // should be enough to keep the average text file + DIV_init(&tdcv, 0); unsigned int dsi = 0; PyObject* ds = 0; @@ -796,7 +653,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DCV_reserve_memory(&dcv, approx_num_cmds); + DIV_reserve_memory(&dcv, approx_num_cmds); // parse command stream ull tbw = 0; // Amount of target bytes written @@ -829,7 +686,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) break; } - DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); + DC_init(DIV_append(&dcv), tbw, cp_size, cp_off); tbw += cp_size; } else if (cmd) { @@ -862,7 +719,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #endif - DeltaChunk* dc = DCV_append(&dcv); + DeltaChunk* dc = DIV_append(&dcv); DC_init(dc, tbw, num_bytes, 0); // gather the data, or (possibly) share single blocks @@ -892,7 +749,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } if (!is_first_run){ - if (!DCV_connect_with_base(&tdcv, &dcv)){ + if (!DIV_connect_with_base(&tdcv, &dcv)){ error = 1; } } @@ -905,10 +762,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (is_first_run){ tdcv = dcv; // wipe out dcv without destroying the members, get its own memory - DCV_init(&dcv, tdcv.size); + DIV_init(&dcv, tdcv.size); } else { // destroy members, but keep memory - DCV_reset(&dcv); + DIV_reset(&dcv); } loop_end: @@ -931,7 +788,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (dsi > 1){ // otherwise dcv equals tcl - DCV_destroy(&dcv); + DIV_destroy(&dcv); } // Return the actual python object - its just a container @@ -939,7 +796,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); // Otherwise tdcv would be deallocated by the chunk list - DCV_destroy(&tdcv); + DIV_destroy(&tdcv); error = 1; } else { // Plain copy, don't deallocate From 29fe629dfc11103398f97f0886d79a6f59deeb75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2010 18:37:52 +0200 Subject: [PATCH 0106/1790] Intermediate commit, working my way through the code, step by step. Didn't even try to compile it yet --- _delta_apply.c | 183 +++++++++++++++++++++++++++++++++++++------------ stream.py | 13 +--- 2 files changed, 144 insertions(+), 52 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 9768b5264..bd09bc5ed 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -14,6 +14,24 @@ typedef uchar bool; const ull gDIV_grow_by = 100; +// DELTA STREAM ACCESS +/////////////////////// +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; + return size; +} + + // DELTA INFO ///////////// typedef struct { @@ -22,6 +40,65 @@ typedef struct { } DeltaInfo; +// TOP LEVEL STREAM INFO +///////////////////////////// +typedef struct { + const uchar* tds; + Py_ssize_t* tdslen; + Py_ssize_t target_size; // size of the target buffer which can hold all data + PyObject* parent_object; +} ToplevelStreamInfo; + + +void TSI_init(ToplevelStreamInfo* info) +{ + info->tds = 0; + info->tdslen = 0; + info->target_size = 0; + info->parent_object = 0; + +} + +void TSI_destroy(ToplevelStreamInfo* info) +{ + if (info->parent_object){ + Py_DECREF(info->parent_object); + info->parent_object = 0; + } else if (info->tds){ + PyMem_Free(info->tds); + } +} + +// initialize our set stream to point to the first chunk +// Fill in the header information, which is the base and target size +void TSI_init_stream(ToplevelStreamInfo* info) +{ + assert(info->tds && info->tdslen) + + // init stream + const uchar* tdsend = info->tds + info->tdslen; + msb_size(&info->tds, tdsend); + info->target_size = msb_size(&info->tds, tdsend); +} + +// duplicate the data currently owned by the parent object drop its refcount +// return 1 on success +bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) +{ + assert(info.parent_object); + + uchar* ptmp = PyMem_Malloc(info.tdslen); + if (!ptmp){ + return 0; + } + memcpy((void*)ptmp, info.tds, info.tdslen); + tds = ptmp; + Py_DECREF(info.parent_object); + info.parent_object = 0; + + return 1; +} + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -452,7 +529,7 @@ bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) typedef struct { PyObject_HEAD // ----------- - DeltaInfoVector vec; + ToplevelStreamInfo istream; } DeltaChunkList; @@ -465,34 +542,20 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - DIV_init(&self->vec, 0); + TSI_init(&self->istream, 0); return 0; } static void DCL_dealloc(DeltaChunkList* self) { - DIV_destroy(&(self->vec)); -} - -static -PyObject* DCL_len(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DIV_len(&self->vec)); -} - -static inline -ull DCL_rbound(DeltaChunkList* self) -{ - if (DIV_empty(&self->vec)) - return 0; - return DIV_rbound(&self->vec); + TSI_destroy(&(self->istream)); } static PyObject* DCL_py_rbound(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(DCL_rbound(self)); + return PyLong_FromUnsignedLongLong(self->istream->target_size); } // Write using a write function, taking remaining bytes from a base buffer @@ -535,7 +598,6 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -596,21 +658,6 @@ DeltaChunkList* DCL_new_instance(void) return dcl; } -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator @@ -626,22 +673,71 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } DeltaInfoVector dcv; - DeltaInfoVector tdcv; + ToplevelStreamInfo tdsinfo; + TSI_init(&tdsinfo); DIV_init(&dcv, 100); // should be enough to keep the average text file - DIV_init(&tdcv, 0); - unsigned int dsi = 0; - PyObject* ds = 0; + + // GET TOPLEVEL DELTA STREAM int error = 0; - for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + PyObject* ds = 0; + unsigned int dsi = 0; + ds = PyIter_Next(stream_iter); + if (!ds){ + error = 1; + goto _error; + } + + dsi += 1; + tdsinfo.parent_object = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(tdsinfo.parent_object)){ + Py_DECREF(ds); + error = 1; + goto _error; + } + + PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); + if (tdslen > pow(2, 32)){ + // parent object is deallocated by info structure + Py_DECREF(ds); + PyErr_SetString(PyExc_RuntimeError("Cannot handle deltas larger than 4GB")); + tdsinfo.tdb = 0; + + error = 1; + goto _error; + } + Py_DECREF(ds); + + // INTEGRATE ANCESTOR DELTA STREAMS + PyObject* db = 0; + TSI_init_stream(&tdsinfo, tdb); + + + for (ds = PyIter_Next(stream_iter); ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) { - PyObject* db = PyObject_CallMethod(ds, "read", 0); + // Its important to initialize this before the next block which can jump + // to code who needs this to exist ! + PyObject* db = 0; + + // When processing the first delta, we know we will have to alter the tds + // Hence we copy it and deallocate the parent object + if (ds == 1) { + if (!TSI_copy_stream_from_object(&tdsinfo)){ + PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); + // info structure takes care of the parent_object + error = 1; + goto loop_end; + } + } + + db = PyObject_CallMethod(ds, "read", 0); if (!PyObject_CheckReadBuffer(db)){ error = 1; PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); goto loop_end; } + // Fill the stream info structure const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); @@ -778,10 +874,13 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } }// END for each stream object - if (dsi == 0 && ! error){ + if (dsi == 0){ PyErr_SetString(PyExc_ValueError, "No streams provided"); } + +_error: + if (stream_iter != dstreams){ Py_DECREF(stream_iter); } @@ -800,7 +899,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; } else { // Plain copy, don't deallocate - dcl->vec = tdcv; + dcl->istream = tdsinfo; } if (error){ diff --git a/stream.py b/stream.py index 38c86dae7..0d8972898 100644 --- a/stream.py +++ b/stream.py @@ -349,7 +349,7 @@ def _set_cache_too_slow_without_c(self, attr): # call len directly, as the (optional) c version doesn't implement the sequence # protocol - if dcl.__len__() == 0: + if dcl.rbound() == 0: self._size = 0 self._mm_target = allocate_memory(0) return @@ -367,15 +367,6 @@ def _set_cache_too_slow_without_c(self, attr): self._mm_target.seek(0) - def _set_cache_(self, attr): - """Determine which version to use depending on the configuration of the deltas - :note: we are only called if we have the performance module""" - # otherwise it depends on the amount of memory to shift around - if len(self._dstreams) > 1 and self._bstream.size < 150000: - return self._set_cache_too_slow_without_c(attr) - else: - return self._set_cache_brute_(attr) - def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" @@ -456,6 +447,8 @@ def _set_cache_brute_(self, attr): #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c #} END configuration From 2414fd48969bf5bff510c8ded83edd7b4900626b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Oct 2010 21:06:17 +0200 Subject: [PATCH 0107/1790] Brutally made code compile, most of the major functions are still commented out, but it should just be a matter of time until its back and working --- _delta_apply.c | 324 ++++++++++++++++++++++--------------------------- 1 file changed, 142 insertions(+), 182 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index bd09bc5ed..abeaed5f1 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -32,31 +32,24 @@ ull msb_size(const uchar** datap, const uchar* top) } -// DELTA INFO -///////////// -typedef struct { - uint dso; // delta stream offset - uint to; // target offset (cache) -} DeltaInfo; - - // TOP LEVEL STREAM INFO ///////////////////////////// typedef struct { - const uchar* tds; - Py_ssize_t* tdslen; - Py_ssize_t target_size; // size of the target buffer which can hold all data - PyObject* parent_object; + const uchar *tds; + Py_ssize_t tdslen; // size of tds in bytes + Py_ssize_t target_size; // size of the target buffer which can hold all data + uint numChunks; // amount of chunks in the delta stream + PyObject *parent_object; } ToplevelStreamInfo; void TSI_init(ToplevelStreamInfo* info) { - info->tds = 0; + info->tds = NULL; info->tdslen = 0; + info->numChunks = 0; info->target_size = 0; info->parent_object = 0; - } void TSI_destroy(ToplevelStreamInfo* info) @@ -65,7 +58,7 @@ void TSI_destroy(ToplevelStreamInfo* info) Py_DECREF(info->parent_object); info->parent_object = 0; } else if (info->tds){ - PyMem_Free(info->tds); + PyMem_Free((void*)info->tds); } } @@ -73,7 +66,7 @@ void TSI_destroy(ToplevelStreamInfo* info) // Fill in the header information, which is the base and target size void TSI_init_stream(ToplevelStreamInfo* info) { - assert(info->tds && info->tdslen) + assert(info->tds && info->tdslen); // init stream const uchar* tdsend = info->tds + info->tdslen; @@ -85,16 +78,16 @@ void TSI_init_stream(ToplevelStreamInfo* info) // return 1 on success bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) { - assert(info.parent_object); + assert(info->parent_object); - uchar* ptmp = PyMem_Malloc(info.tdslen); + uchar* ptmp = PyMem_Malloc(info->tdslen); if (!ptmp){ return 0; } - memcpy((void*)ptmp, info.tds, info.tdslen); - tds = ptmp; - Py_DECREF(info.parent_object); - info.parent_object = 0; + memcpy((void*)ptmp, info->tds, info->tdslen); + info->tds = ptmp; + Py_DECREF(info->parent_object); + info->parent_object = 0; return 1; } @@ -152,11 +145,21 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec // DELTA CHUNK VECTOR ///////////////////// + +// DELTA INFO +///////////// +typedef struct { + uint dso; // delta stream offset + uint to; // target offset (cache) +} DeltaInfo; + + typedef struct { - DeltaInfo* mem; // Memory - const uchar* dstream; // pointer to delta stream we index - Py_ssize_t size; // Size in DeltaInfos - Py_ssize_t reserved_size; // Reserve in DeltaInfos + DeltaInfo *mem; // Memory + uint di_last_size; // size of the last element - we can't compute it using the next bound + const uchar *dstream; // pointer to delta stream we index - its borrowed + Py_ssize_t size; // Amount of DeltaInfos + Py_ssize_t reserved_size; // Reserved amount of DeltaInfos } DeltaInfoVector; @@ -164,7 +167,7 @@ typedef struct { // Reserve enough memory to hold the given amount of delta chunks // Return 1 on success // NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DCVs change often, and reallocs are expensive +// just for a single additional entry. DIVs change often, and reallocs are expensive inline int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) { @@ -219,18 +222,19 @@ int DIV_init(DeltaInfoVector* vec, ull initial_size) vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; + vec->di_last_size = 0; return DIV_grow_by(vec, initial_size); } inline -ull DIV_len(const DeltaInfoVector* vec) +Py_ssize_t DIV_len(const DeltaInfoVector* vec) { return vec->size; } inline -ull DIV_lbound(const DeltaInfoVector* vec) +uint DIV_lbound(const DeltaInfoVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -251,18 +255,6 @@ DeltaInfo* DIV_last(const DeltaInfoVector* vec) return DIV_get(vec, vec->size-1); } -inline -ull DIV_rbound(const DeltaInfoVector* vec) -{ - return DC_rbound(DIV_last(vec)); -} - -inline -ull DIV_size(const DeltaInfoVector* vec) -{ - return DIV_rbound(vec) - DIV_lbound(vec); -} - inline int DIV_empty(const DeltaInfoVector* vec) { @@ -285,19 +277,24 @@ DeltaInfo* DIV_first(const DeltaInfoVector* vec) return vec->mem; } +// return rbound offset in bytes. We use information contained in the +// vec to do that +inline +uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) +{ + if (DIV_last(vec) == di){ + return di->to + vec->di_last_size; + } else { + return (di+1)->to; + } +} + void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG fprintf(stderr, "Freeing %p\n", (void*)vec->mem); #endif - - const DeltaInfo* end = &vec->mem[vec->size]; - DeltaInfo* i; - for(i = vec->mem; i < end; i++){ - DC_destroy(i); - } - PyMem_Free(vec->mem); vec->size = 0; vec->reserved_size = 0; @@ -313,21 +310,13 @@ void DIV_forget_members(DeltaInfoVector* vec) vec->size = 0; } -// Reset the vector so that its size will be zero, and its members will -// have been deallocated properly. +// Reset the vector so that its size will be zero // It will keep its memory though, and hence can be filled again inline void DIV_reset(DeltaInfoVector* vec) { if (vec->size == 0) return; - - DeltaInfo* dc = DIV_first(vec); - const DeltaInfo* dcend = DIV_end(vec); - for(;dc < dcend; dc++){ - DC_destroy(dc); - } - vec->size = 0; } @@ -363,7 +352,7 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) dc = vec->mem + mid; if (dc->to > ofs){ hi = mid; - } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + } else if ((DIV_info_rbound(vec, dc) > ofs) | (dc->to == ofs)) { return dc; } else { lo = mid + 1; @@ -378,7 +367,7 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) inline uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) { - uint num_dc = 0; + /*uint num_dc = 0; DeltaInfo* cdc = DIV_closest_chunk(src, ofs); // partial overlap @@ -404,7 +393,9 @@ uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) } } - return num_dc; + return num_dc;*/ + assert(0); // TODO + return 0; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -414,8 +405,9 @@ uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) { + /* assert(DIV_lbound(src) <= ofs); - assert((ofs + size) <= DIV_rbound(src)); + assert((ofs + size) <= DIV_last(src)->to + src->di_last_size); DeltaInfo* cdc = DIV_closest_chunk(src, ofs); uint num_chunks = 0; @@ -450,13 +442,17 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull assert(size == 0); return num_chunks; + */ + assert(0); // TODO + return 0; } -// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. -bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) +// Take slices of div into the corresponding area of the tsi, which is the topmost +// delta to apply. +bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { + /* uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; @@ -521,6 +517,9 @@ bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) PyMem_Free(offset_array); return 1; + */ + assert(0); // TODO + return 0; } // DELTA CHUNK LIST (PYTHON) @@ -542,7 +541,7 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - TSI_init(&self->istream, 0); + TSI_init(&self->istream); return 0; } @@ -555,13 +554,14 @@ void DCL_dealloc(DeltaChunkList* self) static PyObject* DCL_py_rbound(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(self->istream->target_size); + return PyLong_FromUnsignedLongLong(self->istream.target_size); } // Write using a write function, taking remaining bytes from a base buffer static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + /* PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -593,6 +593,9 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) } Py_DECREF(tmpargs); + */ + // TODO + assert(0); Py_RETURN_NONE; } @@ -653,11 +656,51 @@ DeltaChunkList* DCL_new_instance(void) assert(dcl); DCL_init(dcl, 0, 0); - assert(dcl->vec.size == 0); - assert(dcl->vec.mem == NULL); return dcl; } +// Read the next delta chunk from the given stream and advance it +// dc will contain the parsed information, its offset must be set by +// the previous call of next_delta_info, which implies it should remain the +// same instance between the calls. +// Return 1 on success, 0 on failure +inline +bool next_delta_info(const uchar** dstream, DeltaChunk* dc) +{ + const uchar* data = *dstream; + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + dc->to += dc->ts; + dc->data = 0; + dc->so = cp_off; + dc->ts = cp_size; + + } else if (cmd) { + // Just share the data + dc->to += dc->ts; + dc->data = data; + dc->ts = cmd; + dc->so = 0; + } else { + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + return 0; + } + + return 1; +} + static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator @@ -672,16 +715,16 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaInfoVector dcv; + DeltaInfoVector div; ToplevelStreamInfo tdsinfo; TSI_init(&tdsinfo); - DIV_init(&dcv, 100); // should be enough to keep the average text file + DIV_init(&div, 100); // should be enough to keep the average text file // GET TOPLEVEL DELTA STREAM int error = 0; PyObject* ds = 0; - unsigned int dsi = 0; + unsigned int dsi = 0; // delta stream index we process ds = PyIter_Next(stream_iter); if (!ds){ error = 1; @@ -697,11 +740,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); - if (tdslen > pow(2, 32)){ + if (tdsinfo.tdslen > pow(2, 32)){ // parent object is deallocated by info structure Py_DECREF(ds); - PyErr_SetString(PyExc_RuntimeError("Cannot handle deltas larger than 4GB")); - tdsinfo.tdb = 0; + PyErr_SetString(PyExc_RuntimeError, "Cannot handle deltas larger than 4GB"); + tdsinfo.parent_object = 0; error = 1; goto _error; @@ -709,11 +752,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(ds); // INTEGRATE ANCESTOR DELTA STREAMS - PyObject* db = 0; - TSI_init_stream(&tdsinfo, tdb); + TSI_init_stream(&tdsinfo); - for (ds = PyIter_Next(stream_iter); ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) { // Its important to initialize this before the next block which can jump // to code who needs this to exist ! @@ -721,7 +763,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // When processing the first delta, we know we will have to alter the tds // Hence we copy it and deallocate the parent object - if (ds == 1) { + if (dsi == 1) { if (!TSI_copy_stream_from_object(&tdsinfo)){ PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); // info structure takes care of the parent_object @@ -744,125 +786,45 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* dend = data + dlen; // read header - const ull base_size = msb_size(&data, dend); + msb_size(&data, dend); const ull target_size = msb_size(&data, dend); // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DIV_reserve_memory(&dcv, approx_num_cmds); + DIV_reserve_memory(&div, approx_num_cmds); // parse command stream - ull tbw = 0; // Amount of target bytes written - bool is_shared_data = dsi != 0; - bool is_first_run = dsi == 0; + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); while (data < dend) { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - const unsigned long rbound = cp_off + cp_size; - if (rbound < cp_size || - rbound > base_size){ - // this really shouldn't happen - error = 1; - assert(0); - break; - } - - DC_init(DIV_append(&dcv), tbw, cp_size, cp_off); - tbw += cp_size; - - } else if (cmd) { - // Compression reduces fragmentation though, which is why we do it - // in all cases. - // It makes the more sense the more consecutive add-chunks we have, - // its more likely in big deltas, for big binary files - const uchar* add_start = data - 1; - const uchar* add_end = dend; - ull num_bytes = cmd; - data += cmd; - ull num_chunks = 1; - while (data < dend){ - //while (0){ - const char c = *data; - if (c & 0x80){ - add_end = data; - break; - } else { - data += 1 + c; // advance by 1 to skip add cmd - num_bytes += c; - num_chunks += 1; - } - } - - #ifdef DEBUG - assert(add_end - add_start > 0); - if (num_chunks > 1){ - fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); - } - #endif - - DeltaChunk* dc = DIV_append(&dcv); - DC_init(dc, tbw, num_bytes, 0); - - // gather the data, or (possibly) share single blocks - if (num_chunks > 1){ - uchar* dcdata = PyMem_Malloc(num_bytes); - while (add_start < add_end){ - const char bytes = *add_start++; - memcpy((void*)dcdata, (void*)add_start, bytes); - dcdata += bytes; - add_start += bytes; - } - DC_set_data_with_ownership(dc, dcdata-num_bytes); - } else { - DC_set_data(dc, data - cmd, cmd, is_shared_data); - } - - tbw += num_bytes; - } else { + if (next_delta_info(&data, &dc)){ + // TODO + assert(0); + } else { error = 1; - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); goto loop_end; } }// END handle command opcodes - if (tbw != target_size){ + + if (DC_rbound(&dc) != target_size){ PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); error = 1; } - if (!is_first_run){ - if (!DIV_connect_with_base(&tdcv, &dcv)){ - error = 1; - } + if (!DIV_connect_with_base(&tdsinfo, &div)){ + error = 1; } - + #ifdef DEBUG - fprintf(stderr, "tdcv->size = %i, tdcv->reserved_size = %i\n", (int)tdcv.size, (int)tdcv.reserved_size); - fprintf(stderr, "dcv->size = %i, dcv->reserved_size = %i\n", (int)dcv.size, (int)dcv.reserved_size); + fprintf(stderr, "tdsinfo->len = %i\n", (int)tdsinfo.tdslen); + fprintf(stderr, "div->size = %i, div->reserved_size = %i\n", (int)div.size, (int)div.reserved_size); #endif - if (is_first_run){ - tdcv = dcv; - // wipe out dcv without destroying the members, get its own memory - DIV_init(&dcv, tdcv.size); - } else { - // destroy members, but keep memory - DIV_reset(&dcv); - } + // destroy members, but keep memory + DIV_reset(&div); loop_end: // perform cleanup @@ -885,20 +847,18 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - if (dsi > 1){ - // otherwise dcv equals tcl - DIV_destroy(&dcv); - } + + DIV_destroy(&div); // Return the actual python object - its just a container DeltaChunkList* dcl = DCL_new_instance(); if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdcv would be deallocated by the chunk list - DIV_destroy(&tdcv); + // Otherwise tdsinfo would be deallocated by the chunk list + TSI_destroy(&tdsinfo); error = 1; } else { - // Plain copy, don't deallocate + // Plain copy, transfer ownership to dcl dcl->istream = tdsinfo; } From 9e62c5481fefcc9e8adf0b6387952e214295223c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 00:52:43 +0200 Subject: [PATCH 0108/1790] Worked my way up to re-encoding delta chunks, connect_with method still needs some work, intermediate commit --- _delta_apply.c | 309 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 243 insertions(+), 66 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index abeaed5f1..d35cb7ebc 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -8,6 +8,7 @@ typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; +typedef unsigned short ushort; typedef uchar bool; // Constants @@ -35,10 +36,11 @@ ull msb_size(const uchar** datap, const uchar* top) // TOP LEVEL STREAM INFO ///////////////////////////// typedef struct { - const uchar *tds; + const uchar *tds; // Toplevel delta stream + const uchar *cstart; // start of the chunks Py_ssize_t tdslen; // size of tds in bytes Py_ssize_t target_size; // size of the target buffer which can hold all data - uint numChunks; // amount of chunks in the delta stream + uint num_chunks; // amount of chunks in the delta stream PyObject *parent_object; } ToplevelStreamInfo; @@ -46,8 +48,9 @@ typedef struct { void TSI_init(ToplevelStreamInfo* info) { info->tds = NULL; + info->cstart = NULL; info->tdslen = 0; - info->numChunks = 0; + info->num_chunks = 0; info->target_size = 0; info->parent_object = 0; } @@ -62,18 +65,37 @@ void TSI_destroy(ToplevelStreamInfo* info) } } +inline +const uchar* TSI_end(ToplevelStreamInfo* info) +{ + return info->tds + info->tdslen; +} + +inline +const uchar* TSI_first(ToplevelStreamInfo* info) +{ + return info->cstart; +} + +// set the stream, and initialize it // initialize our set stream to point to the first chunk // Fill in the header information, which is the base and target size -void TSI_init_stream(ToplevelStreamInfo* info) +inline +void TSI_set_stream(ToplevelStreamInfo* info, const uchar* stream) { + info->tds = stream; + info->cstart = stream; + assert(info->tds && info->tdslen); // init stream - const uchar* tdsend = info->tds + info->tdslen; - msb_size(&info->tds, tdsend); - info->target_size = msb_size(&info->tds, tdsend); + const uchar* tdsend = TSI_end(info); + msb_size(&info->cstart, tdsend); // base size + info->target_size = msb_size(&info->cstart, tdsend); } + + // duplicate the data currently owned by the parent object drop its refcount // return 1 on success bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) @@ -86,12 +108,29 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) } memcpy((void*)ptmp, info->tds, info->tdslen); info->tds = ptmp; + info->cstart = ptmp; Py_DECREF(info->parent_object); info->parent_object = 0; return 1; } +// make sure we have the given amount of memory available. This will change +// our official length in bytes right away, its up to the caller +// to do something useful with the freed space +// Return true on success +bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) +{ + assert(info->tds); + if (num_bytes <= info->tdslen){ + return 1; + } + info->tds = PyMem_Realloc((void*)info->tds, num_bytes); + info->cstart = info->tds; + + return info->tds != NULL; +} + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -99,8 +138,8 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) // The data pointer is always shared typedef struct { ull to; - ull ts; - ull so; + uint ts; + uint so; const uchar* data; } DeltaChunk; @@ -141,9 +180,66 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec } +// Encode the information in the given delta chunk and write the byte-stream +// into the given output stream +inline +void DC_encode_to(const DeltaChunk* dc, uchar** pout) +{ + uchar* out = *pout; + if (dc->data){ + *out++ = (uchar)dc->ts; + memcpy(out, dc->data, dc->ts); + out += dc->ts; + } else { + uchar i = 0x80; + uchar* op = out++; + uint moff = dc->so; + uint msize = dc->ts; + + if (moff & 0x000000ff) + *out++ = moff >> 0, i |= 0x01; + if (moff & 0x0000ff00) + *out++ = moff >> 8, i |= 0x02; + if (moff & 0x00ff0000) + *out++ = moff >> 16, i |= 0x04; + if (moff & 0xff000000) + *out++ = moff >> 24, i |= 0x08; + + if (msize & 0x00ff) + *out++ = msize >> 0, i |= 0x10; + if (msize & 0xff00) + *out++ = msize >> 8, i |= 0x20; + + *op = i; + } + *pout = out; +} + +// Return: amount of bytes one would need to encode dc +inline +ushort DC_count_encode_bytes(const DeltaChunk* dc) +{ + if (dc->data){ + return 1 + dc->ts; // cmd byte + actual data bytes + } else { + ushort c = 1; // cmd byte + uint ts = dc->ts; + ull to = dc->to; + + // offset + c += to & 0x000000FF; + c += to & 0x0000FF00; + c += to & 0x00FF0000; + c += to & 0xFF000000; + + // size - max size is 0x10000, its encoded with 0 size bits + c += ts & 0x000000FF; + c += ts & 0x0000FF00; + + return c; + } +} -// DELTA CHUNK VECTOR -///////////////////// // DELTA INFO @@ -154,10 +250,13 @@ typedef struct { } DeltaInfo; +// DELTA INFO VECTOR +////////////////////// + typedef struct { - DeltaInfo *mem; // Memory - uint di_last_size; // size of the last element - we can't compute it using the next bound - const uchar *dstream; // pointer to delta stream we index - its borrowed + DeltaInfo *mem; // Memory for delta infos + uint di_last_size; // size of the last element - we can't compute it using the next bound + const uchar *dstream; // borrowed ointer to delta stream we index Py_ssize_t size; // Amount of DeltaInfos Py_ssize_t reserved_size; // Reserved amount of DeltaInfos } DeltaInfoVector; @@ -220,6 +319,7 @@ int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) int DIV_init(DeltaInfoVector* vec, ull initial_size) { vec->mem = NULL; + vec->dstream = NULL; vec->size = 0; vec->reserved_size = 0; vec->di_last_size = 0; @@ -289,6 +389,24 @@ uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) } } +// return size of the given delta info item +inline +uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo const* veclast) +{ + if (veclast == di){ + return vec->di_last_size; + } else { + return (di+1)->to - di->to; + } +} + +// return size of the given delta info item +inline +uint DIV_info_size(const DeltaInfoVector* vec, const DeltaInfo* di) +{ + return DIV_info_size2(vec, di, DIV_last(vec)); +} + void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ @@ -344,16 +462,16 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) ull lo = 0; ull hi = vec->size; ull mid; - DeltaInfo* dc; + DeltaInfo* di; while (lo < hi) { mid = (lo + hi) / 2; - dc = vec->mem + mid; - if (dc->to > ofs){ + di = vec->mem + mid; + if (di->to > ofs){ hi = mid; - } else if ((DIV_info_rbound(vec, dc) > ofs) | (dc->to == ofs)) { - return dc; + } else if ((DIV_info_rbound(vec, di) > ofs) | (di->to == ofs)) { + return di; } else { lo = mid + 1; } @@ -362,40 +480,58 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) return DIV_last(vec); } +// forward declaration +const uchar* next_delta_info(const uchar*, DeltaChunk*); -// Return the amount of chunks a slice at the given spot would have +// Return the amount of chunks a slice at the given spot would have, as well as +// its size in bytes it would have if the possibly partial chunks would be encoded +// The bytes will be added inline -uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) +uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull size, uint* out_bytes) { - /*uint num_dc = 0; - DeltaInfo* cdc = DIV_closest_chunk(src, ofs); + uint num_dc = 0; + DeltaInfo* cdi = DIV_closest_chunk(src, ofs); + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); // partial overlap - if (cdc->to != ofs) { - const ull relofs = ofs - cdc->to; - size -= cdc->ts - relofs < size ? cdc->ts - relofs : size; + if (cdi->to != ofs) { + const ull relofs = ofs - cdi->to; + const uint cdisize = DIV_info_size(src, cdi); + size -= cdisize - relofs < size ? cdisize - relofs : size; num_dc += 1; - cdc += 1; + cdi += 1; + + // get the size in bytes the info would have + next_delta_info(src->dstream + cdi->dso, &dc); + *out_bytes += DC_count_encode_bytes(&dc); if (size == 0){ return num_dc; } } - const DeltaInfo* vecend = DIV_end(src); - for( ;(cdc < vecend) && size; ++cdc){ + const DeltaInfo const* vecend = DIV_end(src); + const DeltaInfo const* veclast = DIV_last(src); + for( ;(cdi < vecend) && size; ++cdi){ num_dc += 1; - if (cdc->ts < size) { - size -= cdc->ts; + + const uint cdisize = DIV_info_size2(src, cdi, veclast); + + next_delta_info(src->dstream + cdi->dso, &dc); + *out_bytes += DC_count_encode_bytes(&dc); + + if (cdisize < size) { + size -= cdisize; } else { size = 0; break; } } - return num_dc;*/ - assert(0); // TODO - return 0; + *out_bytes += 0; + return num_dc; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -447,40 +583,47 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull return 0; } - // Take slices of div into the corresponding area of the tsi, which is the topmost // delta to apply. bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { - /* - uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + assert(tsi->num_chunks); + + uint *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(uint)); if (!offset_array){ return 0; } uint* pofs = offset_array; uint num_addchunks = 0; + uint num_addbytes = 0; - DeltaInfo* dc = DIV_first(tdcv); - const DeltaInfo* dcend = DIV_end(tdcv); + const uchar* data = TSI_first(tsi); + const uchar const* dend = TSI_end(tsi); + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); // OFFSET RUN - for (;dc < dcend; dc++, pofs++) + for (;data < dend; pofs++) { // Data chunks don't need processing *pofs = num_addchunks; - if (dc->data){ + data = next_delta_info(data, &dc); + + if (dc.data){ continue; } // offset the next chunk by the amount of chunks in the slice // - 1, because we replace our own chunk - num_addchunks += DIV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + num_addchunks += DIV_count_slice_chunks_and_bytes(div, dc.so, dc.ts, &num_addbytes) - 1; + assert(num_addbytes); } + /* // reserve enough memory to hold all the new chunks // reinit pointers, array could have been reallocated - DIV_reserve_memory(tdcv, tdcv->size + num_addchunks); + TSI_resize(tsis, tsi->tdslen + num_addbytes); dc = DIV_last(tdcv); dcend = DIV_first(tdcv) - 1; @@ -514,12 +657,10 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) tdc->to += relofs; } } - + */ PyMem_Free(offset_array); return 1; - */ - assert(0); // TODO - return 0; + } // DELTA CHUNK LIST (PYTHON) @@ -663,23 +804,22 @@ DeltaChunkList* DCL_new_instance(void) // dc will contain the parsed information, its offset must be set by // the previous call of next_delta_info, which implies it should remain the // same instance between the calls. -// Return 1 on success, 0 on failure +// Return the altered uchar pointer, reassign it to the input data inline -bool next_delta_info(const uchar** dstream, DeltaChunk* dc) +const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) { - const uchar* data = *dstream; const char cmd = *data++; if (cmd & 0x80) { - unsigned long cp_off = 0, cp_size = 0; + uint cp_off = 0, cp_size = 0; if (cmd & 0x01) cp_off = *data++; if (cmd & 0x02) cp_off |= (*data++ << 8); if (cmd & 0x04) cp_off |= (*data++ << 16); if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); if (cmd & 0x10) cp_size = *data++; if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cmd & 0x40) cp_size |= (*data++ << 16); // this should never get hit with current deltas ... if (cp_size == 0) cp_size = 0x10000; dc->to += dc->ts; @@ -695,10 +835,34 @@ bool next_delta_info(const uchar** dstream, DeltaChunk* dc) dc->so = 0; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - return 0; + return NULL; } - return 1; + return data; +} + +// Return amount of chunks encoded in the given delta stream +// If read_header is True, then the header msb chunks will be read first. +// Otherwise, the stream is assumed to be scrubbed one past the header +uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) +{ + // read header + if (read_header){ + msb_size(&data, dend); + msb_size(&data, dend); + } + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + uint num_chunks = 0; + + while (data < dend) + { + data = next_delta_info(data, &dc); + num_chunks += 1; + }// END handle command opcodes + + return num_chunks; } static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) @@ -751,10 +915,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } Py_DECREF(ds); - // INTEGRATE ANCESTOR DELTA STREAMS - TSI_init_stream(&tdsinfo); - + // let it officially know, and initialize its internal state + TSI_set_stream(&tdsinfo, tdsinfo.tds); + // INTEGRATE ANCESTOR DELTA STREAMS for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) { // Its important to initialize this before the next block which can jump @@ -770,6 +934,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; goto loop_end; } + + tdsinfo.num_chunks = compute_chunk_count(tdsinfo.cstart, TSI_end(&tdsinfo), 0); } db = PyObject_CallMethod(ds, "read", 0); @@ -783,37 +949,48 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* dend = data + dlen; + const uchar const* dstart = data; + const uchar const* dend = data + dlen; + div.dstream = dstart; + + if (dlen > pow(2, 32)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Cannot currently handle deltas larger than 4GB"); + goto loop_end; + } - // read header + // READ HEADER msb_size(&data, dend); const ull target_size = msb_size(&data, dend); - // Assume good compression for the adds - const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DIV_reserve_memory(&div, approx_num_cmds); + DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); // parse command stream DeltaChunk dc; + DeltaInfo* di = 0; // temporary pointer DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); while (data < dend) { - if (next_delta_info(&data, &dc)){ - // TODO - assert(0); + di = DIV_append(&div); + di->dso = data - dstart; + if ((data = next_delta_info(data, &dc))){ + di->to = dc.to; } else { error = 1; goto loop_end; } }// END handle command opcodes + // finalize information + div.di_last_size = dc.ts; + if (DC_rbound(&dc) != target_size){ PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); error = 1; } - + if (!DIV_connect_with_base(&tdsinfo, &div)){ error = 1; } From 8693a7e37af03cd5d36f36fcc5ed01e938798905 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 12:26:07 +0200 Subject: [PATCH 0109/1790] Implemented connect_with which includes all the slicing functions, which now operate on the delta stream data directly, its yet to be tested though, and I am afraid of this --- _delta_apply.c | 207 ++++++++++++++++++++++++++++--------------------- 1 file changed, 118 insertions(+), 89 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index d35cb7ebc..5f1d9c379 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -106,9 +106,12 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) if (!ptmp){ return 0; } + uint ofs = (uint)(info->cstart - info->tds); memcpy((void*)ptmp, info->tds, info->tdslen); + info->tds = ptmp; - info->cstart = ptmp; + info->cstart = ptmp + ofs; + Py_DECREF(info->parent_object); info->parent_object = 0; @@ -125,8 +128,10 @@ bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) if (num_bytes <= info->tdslen){ return 1; } + uint ofs = (uint)(info->cstart - info->tds); info->tds = PyMem_Realloc((void*)info->tds, num_bytes); - info->cstart = info->tds; + info->tdslen = num_bytes; + info->cstart = info->tds + ofs; return info->tds != NULL; } @@ -182,19 +187,21 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec // Encode the information in the given delta chunk and write the byte-stream // into the given output stream +// It will be copied into the given bounds, the given size must be the final size +// and work with the given relative offset - hence the bounds are assumed to be +// correct and to fit within the unaltered dc inline -void DC_encode_to(const DeltaChunk* dc, uchar** pout) +void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; if (dc->data){ - *out++ = (uchar)dc->ts; - memcpy(out, dc->data, dc->ts); - out += dc->ts; + *out++ = (uchar)size; + memcpy(out, dc->data+ofs, size); + out += size; } else { uchar i = 0x80; uchar* op = out++; - uint moff = dc->so; - uint msize = dc->ts; + uint moff = dc->so+ofs; if (moff & 0x000000ff) *out++ = moff >> 0, i |= 0x01; @@ -205,10 +212,10 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout) if (moff & 0xff000000) *out++ = moff >> 24, i |= 0x08; - if (msize & 0x00ff) - *out++ = msize >> 0, i |= 0x10; - if (msize & 0xff00) - *out++ = msize >> 8, i |= 0x20; + if (size & 0x00ff) + *out++ = size >> 0, i |= 0x10; + if (size & 0xff00) + *out++ = size >> 8, i |= 0x20; *op = i; } @@ -224,13 +231,13 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) } else { ushort c = 1; // cmd byte uint ts = dc->ts; - ull to = dc->to; + ull so = dc->so; // offset - c += to & 0x000000FF; - c += to & 0x0000FF00; - c += to & 0x00FF0000; - c += to & 0xFF000000; + c += so & 0x000000FF; + c += so & 0x0000FF00; + c += so & 0x00FF0000; + c += so & 0xFF000000; // size - max size is 0x10000, its encoded with 0 size bits c += ts & 0x000000FF; @@ -485,13 +492,15 @@ const uchar* next_delta_info(const uchar*, DeltaChunk*); // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded -// The bytes will be added +// and added to the spot marked by sdc inline -uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull size, uint* out_bytes) +uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) { - uint num_dc = 0; + uint num_bytes = 0; DeltaInfo* cdi = DIV_closest_chunk(src, ofs); + + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -499,63 +508,72 @@ uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull s if (cdi->to != ofs) { const ull relofs = ofs - cdi->to; const uint cdisize = DIV_info_size(src, cdi); - size -= cdisize - relofs < size ? cdisize - relofs : size; - num_dc += 1; - cdi += 1; + const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + size -= actual_size; // get the size in bytes the info would have next_delta_info(src->dstream + cdi->dso, &dc); - *out_bytes += DC_count_encode_bytes(&dc); + dc.so += relofs; + dc.ts = actual_size; + num_bytes += DC_count_encode_bytes(&dc); + + cdi += 1; if (size == 0){ - return num_dc; + return num_bytes; } } const DeltaInfo const* vecend = DIV_end(src); - const DeltaInfo const* veclast = DIV_last(src); - for( ;(cdi < vecend) && size; ++cdi){ - num_dc += 1; - - const uint cdisize = DIV_info_size2(src, cdi, veclast); - + for( ;cdi < vecend; ++cdi){ next_delta_info(src->dstream + cdi->dso, &dc); - *out_bytes += DC_count_encode_bytes(&dc); - if (cdisize < size) { - size -= cdisize; + if (dc.ts < size) { + num_bytes += DC_count_encode_bytes(&dc); + size -= dc.ts; } else { + dc.ts = size; + num_bytes += DC_count_encode_bytes(&dc); size = 0; break; } } - *out_bytes += 0; - return num_dc; + assert(size == 0); + return num_bytes; } // Write a slice as defined by its absolute offset in bytes and its size into the given -// destination memory. The individual chunks written will be a deep copy of the source -// data chunks +// destination memory. The individual chunks written will be a byte copy of the source +// data chunk stream // Return: number of chunks in the slice inline -uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { - /* - assert(DIV_lbound(src) <= ofs); - assert((ofs + size) <= DIV_last(src)->to + src->di_last_size); + assert(DIV_lbound(src) <= tofs); + assert((tofs + size) <= DIV_last(src)->to + src->di_last_size); - DeltaInfo* cdc = DIV_closest_chunk(src, ofs); + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + + DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; // partial overlap - if (cdc->to != ofs) { - const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); - cdc += 1; - size -= dest->ts; - dest += 1; // must be here, we are reading the size ! + if (cdi->to != tofs) { + const uint relofs = tofs - cdi->to; + next_delta_info(src->dstream + cdi->dso, &dc); + const uint cdisize = dc.ts; + const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + + size -= actual_size; + + // adjust dc proportions + + DC_encode_to(&dc, &dest, relofs, actual_size); + num_chunks += 1; + cdi += 1; if (size == 0){ return num_chunks; @@ -563,14 +581,18 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull } const DeltaInfo* vecend = DIV_end(src); - for( ;(cdc < vecend) && size; ++cdc) + for( ;cdi < vecend; ++cdi) { num_chunks += 1; - if (cdc->ts < size) { - DC_copy_to(cdc, dest++); - size -= cdc->ts; + next_delta_info(src->dstream + cdi->dso, &dc); + if (dc.ts < size) { + // Full copy would be possible, but the final length of the dstream + // needs to be used as well to know how many bytes to copy + // TODO: make a DIV_ function for this + DC_encode_to(&dc, &dest, 0, dc.ts); + size -= dc.ts; } else { - DC_offset_copy_to(cdc, dest, 0, size); + DC_encode_to(&dc, &dest, 0, size); size = 0; break; } @@ -578,36 +600,44 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull assert(size == 0); return num_chunks; - */ - assert(0); // TODO - return 0; } + // Take slices of div into the corresponding area of the tsi, which is the topmost // delta to apply. bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { assert(tsi->num_chunks); - uint *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(uint)); + typedef struct { + uint bofs; // byte-offset of delta stream + uint dofs; // delta stream offset relative to tsi->cstart + } OffsetInfo; + + + OffsetInfo *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(OffsetInfo)); if (!offset_array){ return 0; } - uint* pofs = offset_array; - uint num_addchunks = 0; + OffsetInfo* pofs = offset_array; uint num_addbytes = 0; const uchar* data = TSI_first(tsi); + const uchar* prev_data = data; const uchar const* dend = TSI_end(tsi); + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); // OFFSET RUN - for (;data < dend; pofs++) + for (;data < dend; pofs++, prev_data = data) { + + pofs->bofs = num_addbytes; + pofs->dofs = (uint)(prev_data - data); + // Data chunks don't need processing - *pofs = num_addchunks; data = next_delta_info(data, &dc); if (dc.data){ @@ -615,49 +645,48 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } // offset the next chunk by the amount of chunks in the slice - // - 1, because we replace our own chunk - num_addchunks += DIV_count_slice_chunks_and_bytes(div, dc.so, dc.ts, &num_addbytes) - 1; - assert(num_addbytes); + // - N, because we replace our own chunk's bytes + num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } - /* - // reserve enough memory to hold all the new chunks - // reinit pointers, array could have been reallocated - TSI_resize(tsis, tsi->tdslen + num_addbytes); - dc = DIV_last(tdcv); - dcend = DIV_first(tdcv) - 1; - // now, that we have our pointers with the old size - tdcv->size += num_addchunks; + + // reserve enough memory to hold all the new chunks + TSI_resize(tsi, tsi->tdslen + num_addbytes); + const OffsetInfo const* pofs_start = offset_array - 1; + const OffsetInfo* cpofs; + uchar* ds; // pointer into the delta stream + const uchar* nds; // next pointer, used for size retrieving the size + uint num_addchunks = 0; // total amount of chunks added // Insert slices, from the end to the beginning, which allows memcpy // to be used, with a little help of the offset array - for (pofs -= 1; dc > dcend; dc--, pofs-- ) + for (cpofs = pofs - 1; cpofs > pofs_start; cpofs--) { + ds = (uchar*)(tsi->cstart + cpofs->dofs); + nds = next_delta_info(ds, &dc); + // Data chunks don't need processing - const uint ofs = *pofs; - if (dc->data){ + if (dc.data){ // NOTE: could peek the preceeding chunks to figure out whether they are // all just moved by ofs. In that case, they can move as a whole! // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort - if (ofs){ - memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaInfo)); + if (pofs->bofs){ + memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; } - // Copy Chunks, and move their target offset into place - // As we could override dc when slicing, we get the data here - const ull relofs = dc->to - dc->so; - - DeltaInfo* tdc = dc + ofs; - DeltaInfo* tdcend = tdc + DIV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); - for(;tdc < tdcend; tdc++){ - tdc->to += relofs; - } + // Copy Chunks - target offset is determined by their location and size + // hence it doesn't need specific adjustment + // -1 chunks because we overwrite our own chunk ( by not copying it ) + num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); + num_addchunks -= 1; } - */ + + tsi->num_chunks += num_addchunks; + PyMem_Free(offset_array); return 1; @@ -823,7 +852,7 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) if (cp_size == 0) cp_size = 0x10000; dc->to += dc->ts; - dc->data = 0; + dc->data = NULL; dc->so = cp_off; dc->ts = cp_size; From c16de40fccf84d066194d2bf95dea56aa76dafd1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 14:34:00 +0200 Subject: [PATCH 0110/1790] Implemented apply - there are still some issues to work out though --- _delta_apply.c | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 5f1d9c379..6d7c4c9d0 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -165,7 +165,6 @@ ull DC_rbound(const DeltaChunk* dc) } // Apply -// TODO: remove, just left it for reference inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) { @@ -252,7 +251,7 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) // DELTA INFO ///////////// typedef struct { - uint dso; // delta stream offset + uint dso; // delta stream offset, relative to the very start of the stream uint to; // target offset (cache) } DeltaInfo; @@ -281,10 +280,6 @@ int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) return 1; } - if (num_dc - vec->reserved_size < 10){ - num_dc += gDIV_grow_by; - } - #ifdef DEBUG bool was_null = vec->mem == NULL; #endif @@ -529,6 +524,8 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { + // TODO: could just count size of the delta chunk in the stream instead + // of reencoding num_bytes += DC_count_encode_bytes(&dc); size -= dc.ts; } else { @@ -731,7 +728,7 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { - /* + PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -749,23 +746,24 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) return NULL; } - const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DIV_end(&self->vec); - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + const uchar* base; + Py_ssize_t baselen; + PyObject_AsReadBuffer(pybuf, (const void**)&base, &baselen); PyObject* tmpargs = PyTuple_New(1); - for(; i < end; i++){ - DC_apply(i, data, writeproc, tmpargs); + const uchar* data = TSI_first(&self->istream); + const uchar const* dend = TSI_end(&self->istream); + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + + while (data < dend){ + data = next_delta_info(data, &dc); + DC_apply(&dc, base, writeproc, tmpargs); } Py_DECREF(tmpargs); - */ - // TODO - assert(0); Py_RETURN_NONE; } @@ -911,7 +909,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaInfoVector div; ToplevelStreamInfo tdsinfo; TSI_init(&tdsinfo); - DIV_init(&div, 100); // should be enough to keep the average text file + DIV_init(&div, 0); // GET TOPLEVEL DELTA STREAM @@ -1020,13 +1018,17 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; } + #ifdef DEBUG + fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + #endif + if (!DIV_connect_with_base(&tdsinfo, &div)){ error = 1; } #ifdef DEBUG - fprintf(stderr, "tdsinfo->len = %i\n", (int)tdsinfo.tdslen); - fprintf(stderr, "div->size = %i, div->reserved_size = %i\n", (int)div.size, (int)div.reserved_size); + fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif // destroy members, but keep memory From dc85535de371762503496313f5e753d61997dcfd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 16:22:59 +0200 Subject: [PATCH 0111/1790] Its much closer to a working state now, but still not quite there. Probably just a small thing --- _delta_apply.c | 72 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 6d7c4c9d0..65e949a04 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -57,6 +57,7 @@ void TSI_init(ToplevelStreamInfo* info) void TSI_destroy(ToplevelStreamInfo* info) { + fprintf(stderr, "TSI_destroy: %p\n", info); if (info->parent_object){ Py_DECREF(info->parent_object); info->parent_object = 0; @@ -164,6 +165,12 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } +inline +void DC_print(const DeltaChunk* dc, const char* prefix) +{ + fprintf(stderr, "%s-dc: to = %i, ts = %i, so = %i, data = %p\n", prefix, (int)dc->to, dc->ts, dc->so, dc->data); +} + // Apply inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) @@ -179,6 +186,8 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec assert(0); } + DC_print(dc, "DC_apply"); + // tuple steals reference, and will take care about the deallocation PyObject_Call(writer, tmpargs, NULL); @@ -233,14 +242,14 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) ull so = dc->so; // offset - c += so & 0x000000FF; - c += so & 0x0000FF00; - c += so & 0x00FF0000; - c += so & 0xFF000000; + c += (so & 0x000000FF) > 0; + c += (so & 0x0000FF00) > 0; + c += (so & 0x00FF0000) > 0; + c += (so & 0xFF000000) > 0; // size - max size is 0x10000, its encoded with 0 size bits - c += ts & 0x000000FF; - c += ts & 0x0000FF00; + c += (ts & 0x000000FF) > 0; + c += (ts & 0x0000FF00) > 0; return c; } @@ -413,7 +422,7 @@ void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG - fprintf(stderr, "Freeing %p\n", (void*)vec->mem); + fprintf(stderr, "DIV_destroy: %p\n", (void*)vec->mem); #endif PyMem_Free(vec->mem); vec->size = 0; @@ -494,8 +503,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) uint num_bytes = 0; DeltaInfo* cdi = DIV_closest_chunk(src, ofs); - - DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -503,13 +510,13 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) if (cdi->to != ofs) { const ull relofs = ofs - cdi->to; const uint cdisize = DIV_info_size(src, cdi); - const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= actual_size; + const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; + size -= max_size; // get the size in bytes the info would have next_delta_info(src->dstream + cdi->dso, &dc); dc.so += relofs; - dc.ts = actual_size; + dc.ts = max_size; num_bytes += DC_count_encode_bytes(&dc); cdi += 1; @@ -547,8 +554,9 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { + fprintf(stderr, "copy slice: ofs = %i, size = %i\n", (int)tofs, size); assert(DIV_lbound(src) <= tofs); - assert((tofs + size) <= DIV_last(src)->to + src->di_last_size); + assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -556,18 +564,22 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; +#ifdef DEBUG + const uchar* deststart = dest; +#endif + // partial overlap if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; next_delta_info(src->dstream + cdi->dso, &dc); const uint cdisize = dc.ts; - const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= actual_size; + size -= max_size; // adjust dc proportions - DC_encode_to(&dc, &dest, relofs, actual_size); + DC_encode_to(&dc, &dest, relofs, max_size); num_chunks += 1; cdi += 1; @@ -580,6 +592,7 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s const DeltaInfo* vecend = DIV_end(src); for( ;cdi < vecend; ++cdi) { + fprintf(stderr, "copy slice: cdi: to = %i, dso = %i\n", (int)cdi->to, (int)cdi->dso); num_chunks += 1; next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { @@ -595,6 +608,10 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s } } +#ifdef DEBUG + fprintf(stderr, "copy slice: Wrote %i bytes\n", (int)(dest - deststart)); +#endif + assert(size == 0); return num_chunks; } @@ -619,6 +636,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) OffsetInfo* pofs = offset_array; uint num_addbytes = 0; + uint dofs = 0; const uchar* data = TSI_first(tsi); const uchar* prev_data = data; @@ -627,16 +645,19 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); + // OFFSET RUN for (;data < dend; pofs++, prev_data = data) { - pofs->bofs = num_addbytes; - pofs->dofs = (uint)(prev_data - data); - - // Data chunks don't need processing data = next_delta_info(data, &dc); + pofs->dofs = dofs; + dofs += (uint)(data-prev_data); + + fprintf(stderr, "pofs->bofs = %i, ->dofs = %i\n", pofs->bofs, pofs->dofs); + DC_print(&dc, "count-run"); + // Data chunks don't need processing if (dc.data){ continue; } @@ -646,6 +667,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } + fprintf(stderr, "num_addbytes = %i\n", num_addbytes); + assert(DC_rbound(&dc) == tsi->target_size); // reserve enough memory to hold all the new chunks @@ -655,6 +678,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) uchar* ds; // pointer into the delta stream const uchar* nds; // next pointer, used for size retrieving the size uint num_addchunks = 0; // total amount of chunks added + DC_init(&dc, 0, 0, 0, NULL); // Insert slices, from the end to the beginning, which allows memcpy // to be used, with a little help of the offset array @@ -682,6 +706,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addchunks -= 1; } + fprintf(stderr, "num_addchunks = %i\n", num_addchunks); tsi->num_chunks += num_addchunks; PyMem_Free(offset_array); @@ -860,6 +885,8 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) dc->data = data; dc->ts = cmd; dc->so = 0; + + data += cmd; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); return NULL; @@ -993,8 +1020,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); // parse command stream - DeltaChunk dc; DeltaInfo* di = 0; // temporary pointer + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); @@ -1019,7 +1046,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #ifdef DEBUG + fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif if (!DIV_connect_with_base(&tdsinfo, &div)){ @@ -1028,7 +1057,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif // destroy members, but keep memory From c077f3f9ac56aa1019273c7ea548ae047e5974f6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 16:28:28 +0200 Subject: [PATCH 0112/1790] now it appears to work completely, still plenty of debug printing though --- _delta_apply.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_delta_apply.c b/_delta_apply.c index 65e949a04..350a2493d 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -693,7 +693,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // all just moved by ofs. In that case, they can move as a whole! // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort - if (pofs->bofs){ + if (cpofs->bofs){ memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; From 28263c97bca171d84b4f4ea022d6638076b903a4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 17:32:31 +0200 Subject: [PATCH 0113/1790] When using it with deeper chains, it can still crash as it can actually (try to) shrink a chunk. This is currently not handled. In that case, we had to virtually move everything x bytes, which should be much like an offset --- _delta_apply.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 350a2493d..2632fbb27 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -57,7 +57,10 @@ void TSI_init(ToplevelStreamInfo* info) void TSI_destroy(ToplevelStreamInfo* info) { +#ifdef DEBUG fprintf(stderr, "TSI_destroy: %p\n", info); +#endif + if (info->parent_object){ Py_DECREF(info->parent_object); info->parent_object = 0; @@ -129,6 +132,10 @@ bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) if (num_bytes <= info->tdslen){ return 1; } + +#ifdef DEBUG + fprintf(stderr, "TSI_resize: to %i bytes\n", num_bytes); +#endif uint ofs = (uint)(info->cstart - info->tds); info->tds = PyMem_Realloc((void*)info->tds, num_bytes); info->tdslen = num_bytes; @@ -186,7 +193,6 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec assert(0); } - DC_print(dc, "DC_apply"); // tuple steals reference, and will take care about the deallocation PyObject_Call(writer, tmpargs, NULL); @@ -554,7 +560,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { - fprintf(stderr, "copy slice: ofs = %i, size = %i\n", (int)tofs, size); assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); @@ -564,10 +569,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; -#ifdef DEBUG - const uchar* deststart = dest; -#endif - // partial overlap if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; @@ -592,7 +593,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s const DeltaInfo* vecend = DIV_end(src); for( ;cdi < vecend; ++cdi) { - fprintf(stderr, "copy slice: cdi: to = %i, dso = %i\n", (int)cdi->to, (int)cdi->dso); num_chunks += 1; next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { @@ -608,10 +608,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s } } -#ifdef DEBUG - fprintf(stderr, "copy slice: Wrote %i bytes\n", (int)(dest - deststart)); -#endif - assert(size == 0); return num_chunks; } @@ -624,7 +620,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->num_chunks); typedef struct { - uint bofs; // byte-offset of delta stream + int bofs; // byte-offset of delta stream uint dofs; // delta stream offset relative to tsi->cstart } OffsetInfo; @@ -635,7 +631,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } OffsetInfo* pofs = offset_array; - uint num_addbytes = 0; + int num_addbytes = 0; uint dofs = 0; const uchar* data = TSI_first(tsi); @@ -651,12 +647,10 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { pofs->bofs = num_addbytes; data = next_delta_info(data, &dc); + assert(data); pofs->dofs = dofs; dofs += (uint)(data-prev_data); - fprintf(stderr, "pofs->bofs = %i, ->dofs = %i\n", pofs->bofs, pofs->dofs); - DC_print(&dc, "count-run"); - // Data chunks don't need processing if (dc.data){ continue; @@ -667,7 +661,12 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } - fprintf(stderr, "num_addbytes = %i\n", num_addbytes); + /* + uint i = 0; + for (; i < tsi->num_chunks; i++){ + fprintf(stderr, "%i: bofs: %i, dofs: %i\n", i, offset_array[i].bofs, offset_array[i].dofs); + } + */ assert(DC_rbound(&dc) == tsi->target_size); @@ -695,6 +694,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // not worth the extra effort if (cpofs->bofs){ memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); + // memmove((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; } @@ -706,7 +706,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addchunks -= 1; } - fprintf(stderr, "num_addchunks = %i\n", num_addchunks); tsi->num_chunks += num_addchunks; PyMem_Free(offset_array); From 1a57dc133ec31c409819b2b40866529ed95a555b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 17:53:12 +0200 Subject: [PATCH 0114/1790] Well, the virtual movement doesn't work - the algorithm really worked only with a fixed chunk size. Now the only chance we have is to allocate an appropriately sized buffer, and work through it directly. This makes things easier, and will make things work \! --- _delta_apply.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 2632fbb27..9df0191e4 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -620,7 +620,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->num_chunks); typedef struct { - int bofs; // byte-offset of delta stream + uint bofs; // byte-offset of delta stream uint dofs; // delta stream offset relative to tsi->cstart } OffsetInfo; @@ -631,7 +631,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } OffsetInfo* pofs = offset_array; - int num_addbytes = 0; + uint num_addbytes = 0; + int bytes = 0; uint dofs = 0; const uchar* data = TSI_first(tsi); @@ -658,15 +659,16 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // offset the next chunk by the amount of chunks in the slice // - N, because we replace our own chunk's bytes - num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); + bytes = DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); + // if we shrink in size, compensate this by moving the start virtually + // + if (bytes < 0){ + fprintf(stderr, "hit negative bytes: %i\n", bytes); + tsi->cstart += abs(bytes); + } + num_addbytes += abs(bytes); } - /* - uint i = 0; - for (; i < tsi->num_chunks; i++){ - fprintf(stderr, "%i: bofs: %i, dofs: %i\n", i, offset_array[i].bofs, offset_array[i].dofs); - } - */ assert(DC_rbound(&dc) == tsi->target_size); @@ -701,8 +703,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // Copy Chunks - target offset is determined by their location and size // hence it doesn't need specific adjustment - // -1 chunks because we overwrite our own chunk ( by not copying it ) num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); + // -1 chunks because we overwrite our own chunk ( by not copying it ) num_addchunks -= 1; } From a7892bc7ebc99d2fe726287cc69716a3494b5ea2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 20:25:01 +0200 Subject: [PATCH 0115/1790] goooosh, it took so long to find a tiny nasty bug ... aarggghhh, lots of debug printing still in there, ... this one better be faster than anything else \! --- _delta_apply.c | 182 ++++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 9df0191e4..71d5b5f8c 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -15,6 +15,7 @@ typedef uchar bool; const ull gDIV_grow_by = 100; + // DELTA STREAM ACCESS /////////////////////// inline @@ -63,10 +64,14 @@ void TSI_destroy(ToplevelStreamInfo* info) if (info->parent_object){ Py_DECREF(info->parent_object); - info->parent_object = 0; + info->parent_object = NULL; } else if (info->tds){ PyMem_Free((void*)info->tds); } + info->tds = NULL; + info->cstart = NULL; + info->tdslen = 0; + info->num_chunks = 0; } inline @@ -122,26 +127,21 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) return 1; } -// make sure we have the given amount of memory available. This will change -// our official length in bytes right away, its up to the caller -// to do something useful with the freed space -// Return true on success -bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) +// Transfer ownership of the given stream into our instance. The amount of chunks +// remains the same, and needs to be set by the caller +void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) { - assert(info->tds); - if (num_bytes <= info->tdslen){ - return 1; - } + assert(info->parent_object == 0); + fprintf(stderr, "TSI_replace_stream\n"); -#ifdef DEBUG - fprintf(stderr, "TSI_resize: to %i bytes\n", num_bytes); -#endif uint ofs = (uint)(info->cstart - info->tds); - info->tds = PyMem_Realloc((void*)info->tds, num_bytes); - info->tdslen = num_bytes; + if (info->tds){ + PyMem_Free((void*)info->tds); + } + info->tds = stream; info->cstart = info->tds + ofs; + info->tdslen = streamlen; - return info->tds != NULL; } // DELTA CHUNK @@ -156,6 +156,9 @@ typedef struct { const uchar* data; } DeltaChunk; +// forward declarations +const uchar* next_delta_info(const uchar*, DeltaChunk*); + inline void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) { @@ -208,6 +211,8 @@ inline void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; + DC_print(dc, "DC_encode_to"); + fprintf(stderr, "DC_encode_to: ofs = %i, size = %i\n" , ofs, size); if (dc->data){ *out++ = (uchar)size; memcpy(out, dc->data+ofs, size); @@ -233,6 +238,18 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) *op = i; } + +#ifdef DEBUG + DeltaChunk mdc; + DC_init(&mdc, 0, 0, 0, NULL); + next_delta_info(*pout, &mdc); + assert(mdc.ts == size); + if (mdc.data) + assert(mdc.data); + else + assert(mdc.so == dc->so+ofs); +#endif + *pout = out; } @@ -497,8 +514,6 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) return DIV_last(vec); } -// forward declaration -const uchar* next_delta_info(const uchar*, DeltaChunk*); // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded @@ -558,10 +573,11 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) // data chunk stream // Return: number of chunks in the slice inline -uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) { assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); + fprintf(stderr, "copy_slice: ofs = %i, size = %i\n", (int)tofs, size); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -573,14 +589,12 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; next_delta_info(src->dstream + cdi->dso, &dc); - const uint cdisize = dc.ts; - const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; + const uint max_size = dc.ts - relofs < size ? dc.ts - relofs : size; size -= max_size; // adjust dc proportions - - DC_encode_to(&dc, &dest, relofs, max_size); + DC_encode_to(&dc, dest, relofs, max_size); num_chunks += 1; cdi += 1; @@ -599,10 +613,10 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s // Full copy would be possible, but the final length of the dstream // needs to be used as well to know how many bytes to copy // TODO: make a DIV_ function for this - DC_encode_to(&dc, &dest, 0, dc.ts); + DC_encode_to(&dc, dest, 0, dc.ts); size -= dc.ts; } else { - DC_encode_to(&dc, &dest, 0, size); + DC_encode_to(&dc, dest, 0, size); size = 0; break; } @@ -619,98 +633,90 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { assert(tsi->num_chunks); - typedef struct { - uint bofs; // byte-offset of delta stream - uint dofs; // delta stream offset relative to tsi->cstart - } OffsetInfo; - - - OffsetInfo *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(OffsetInfo)); - if (!offset_array){ - return 0; - } - - OffsetInfo* pofs = offset_array; - uint num_addbytes = 0; - int bytes = 0; - uint dofs = 0; + uint num_bytes = 0; const uchar* data = TSI_first(tsi); - const uchar* prev_data = data; - const uchar const* dend = TSI_end(tsi); + const uchar* dend = TSI_end(tsi); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); - // OFFSET RUN - for (;data < dend; pofs++, prev_data = data) + // COMPUTE SIZE OF TARGET STREAM + ///////////////////////////////// + for (;data < dend;) { - pofs->bofs = num_addbytes; data = next_delta_info(data, &dc); - assert(data); - pofs->dofs = dofs; - dofs += (uint)(data-prev_data); + DC_print(&dc, "count"); // Data chunks don't need processing if (dc.data){ + num_bytes += 1 + dc.ts; continue; } - // offset the next chunk by the amount of chunks in the slice - // - N, because we replace our own chunk's bytes - bytes = DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); - // if we shrink in size, compensate this by moving the start virtually - // - if (bytes < 0){ - fprintf(stderr, "hit negative bytes: %i\n", bytes); - tsi->cstart += abs(bytes); - } - num_addbytes += abs(bytes); + num_bytes += DIV_count_slice_bytes(div, dc.so, dc.ts); } - assert(DC_rbound(&dc) == tsi->target_size); - // reserve enough memory to hold all the new chunks - TSI_resize(tsi, tsi->tdslen + num_addbytes); - const OffsetInfo const* pofs_start = offset_array - 1; - const OffsetInfo* cpofs; - uchar* ds; // pointer into the delta stream - const uchar* nds; // next pointer, used for size retrieving the size - uint num_addchunks = 0; // total amount of chunks added + // GET NEW DELTA BUFFER + //////////////////////// + uchar *const dstream = PyMem_Malloc(num_bytes); + if (!dstream){ + return 0; + } + + + data = TSI_first(tsi); + const uchar *ndata = data; + dend = TSI_end(tsi); + + uint num_chunks = 0; + uchar* ds = dstream; DC_init(&dc, 0, 0, 0, NULL); - // Insert slices, from the end to the beginning, which allows memcpy - // to be used, with a little help of the offset array - for (cpofs = pofs - 1; cpofs > pofs_start; cpofs--) + // pick slices from the delta and put them into the new stream + for (; data < dend; data = ndata) { - ds = (uchar*)(tsi->cstart + cpofs->dofs); - nds = next_delta_info(ds, &dc); + ndata = next_delta_info(data, &dc); + + DC_print(&dc, "slice"); // Data chunks don't need processing if (dc.data){ - // NOTE: could peek the preceeding chunks to figure out whether they are - // all just moved by ofs. In that case, they can move as a whole! - // tests showed that this is very rare though, even in huge deltas, so its - // not worth the extra effort - if (cpofs->bofs){ - memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); - // memmove((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); - } + // just copy it over + memcpy((void*)ds, (void*)data, ndata - data); + ds += ndata - data; + num_chunks += 1; continue; } - // Copy Chunks - target offset is determined by their location and size - // hence it doesn't need specific adjustment - num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); - // -1 chunks because we overwrite our own chunk ( by not copying it ) - num_addchunks -= 1; + // Copy Chunks + num_chunks += DIV_copy_slice_to(div, &ds, dc.so, dc.ts); } + assert(ds - dstream == num_bytes); + assert(num_chunks >= tsi->num_chunks); + assert(DC_rbound(&dc) == tsi->target_size); + + // finally, replace the streams + TSI_replace_stream(tsi, dstream, num_bytes); + tsi->cstart = dstream; // we have NO header ! + assert(tsi->tds == dstream); + tsi->num_chunks = num_chunks; - tsi->num_chunks += num_addchunks; +#ifdef DEBUG + data = TSI_first(tsi); + dend = TSI_end(tsi); + + DC_init(&dc, 0, 0, 0, NULL); + + while (data < dend){ + data = next_delta_info(data, &dc); + DC_print(&dc, "debug"); + } +#endif - PyMem_Free(offset_array); return 1; } @@ -754,6 +760,7 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + fprintf(stderr, "DCL_apply\n"); PyObject* pybuf = 0; PyObject* writeproc = 0; @@ -890,6 +897,7 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) data += cmd; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + assert(0); return NULL; } @@ -1048,7 +1056,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i, target_size = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen, (int)tdsinfo.target_size); fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif From 81a96250f1758844a1c95f0293083c18c4ce98dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 20:37:39 +0200 Subject: [PATCH 0116/1790] Removed all debug code, it now runs about as fast as the previous version, but with less memory, still slower than the brute force version though --- _delta_apply.c | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 71d5b5f8c..068d89a17 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -132,7 +132,6 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) { assert(info->parent_object == 0); - fprintf(stderr, "TSI_replace_stream\n"); uint ofs = (uint)(info->cstart - info->tds); if (info->tds){ @@ -211,8 +210,6 @@ inline void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; - DC_print(dc, "DC_encode_to"); - fprintf(stderr, "DC_encode_to: ofs = %i, size = %i\n" , ofs, size); if (dc->data){ *out++ = (uchar)size; memcpy(out, dc->data+ofs, size); @@ -239,17 +236,6 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) *op = i; } -#ifdef DEBUG - DeltaChunk mdc; - DC_init(&mdc, 0, 0, 0, NULL); - next_delta_info(*pout, &mdc); - assert(mdc.ts == size); - if (mdc.data) - assert(mdc.data); - else - assert(mdc.so == dc->so+ofs); -#endif - *pout = out; } @@ -577,7 +563,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint { assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); - fprintf(stderr, "copy_slice: ofs = %i, size = %i\n", (int)tofs, size); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -647,7 +632,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) for (;data < dend;) { data = next_delta_info(data, &dc); - DC_print(&dc, "count"); // Data chunks don't need processing if (dc.data){ @@ -681,8 +665,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { ndata = next_delta_info(data, &dc); - DC_print(&dc, "slice"); - // Data chunks don't need processing if (dc.data){ // just copy it over @@ -705,17 +687,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->tds == dstream); tsi->num_chunks = num_chunks; -#ifdef DEBUG - data = TSI_first(tsi); - dend = TSI_end(tsi); - - DC_init(&dc, 0, 0, 0, NULL); - - while (data < dend){ - data = next_delta_info(data, &dc); - DC_print(&dc, "debug"); - } -#endif return 1; @@ -760,8 +731,6 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { - fprintf(stderr, "DCL_apply\n"); - PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -1056,8 +1025,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i, target_size = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen, (int)tdsinfo.target_size); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); + fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i KiB, target_size = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000, (int)tdsinfo.target_size/1000); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i KiB\n", (int)div.size, (int)div.reserved_size, (int)dlen/1000); #endif if (!DIV_connect_with_base(&tdsinfo, &div)){ @@ -1065,7 +1034,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #ifdef DEBUG - fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000); #endif // destroy members, but keep memory From ca829e0b341dd5c3ae1408b24702f2c75db6ec73 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 21:12:42 +0200 Subject: [PATCH 0117/1790] now byte-copying a few chunks where possible when slicing, which gives a few percent of performance --- _delta_apply.c | 20 +++++++++----------- stream.py | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 068d89a17..e99a803be 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -534,13 +534,12 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) } const DeltaInfo const* vecend = DIV_end(src); + const uchar* nstream; for( ;cdi < vecend; ++cdi){ - next_delta_info(src->dstream + cdi->dso, &dc); + nstream = next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { - // TODO: could just count size of the delta chunk in the stream instead - // of reencoding - num_bytes += DC_count_encode_bytes(&dc); + num_bytes += nstream - (src->dstream + cdi->dso); size -= dc.ts; } else { dc.ts = size; @@ -589,16 +588,15 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint } } - const DeltaInfo* vecend = DIV_end(src); - for( ;cdi < vecend; ++cdi) + const uchar* dstream = src->dstream + cdi->dso; + const uchar* nstream = dstream; + for( ; nstream; dstream = nstream) { num_chunks += 1; - next_delta_info(src->dstream + cdi->dso, &dc); + nstream = next_delta_info(dstream, &dc); if (dc.ts < size) { - // Full copy would be possible, but the final length of the dstream - // needs to be used as well to know how many bytes to copy - // TODO: make a DIV_ function for this - DC_encode_to(&dc, dest, 0, dc.ts); + memcpy(*dest, dstream, nstream - dstream); + *dest += nstream - dstream; size -= dc.ts; } else { DC_encode_to(&dc, dest, 0, size); diff --git a/stream.py b/stream.py index 0d8972898..f522bd318 100644 --- a/stream.py +++ b/stream.py @@ -445,7 +445,7 @@ def _set_cache_brute_(self, attr): #{ Configuration - if not has_perf_mod: + if has_perf_mod: _set_cache_ = _set_cache_brute_ else: _set_cache_ = _set_cache_too_slow_without_c From 21e2b5db748f18f70f16cd94ee9fd5adf16ce433 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 21:31:41 +0200 Subject: [PATCH 0118/1790] Updated algorithm paper with the latest changes - thats it for now --- doc/source/algorithm.rst | 18 ++++++++---------- test/performance/test_pack.py | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index 55207b67b..4374cb820 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -58,10 +58,10 @@ GitDB's reverse delta aggregation algorithm =========================================== The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. -In the current implementation, delta streams are parsed into DeltaChunks (->**DC**), which are kept in vectors. Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**). Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. -During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream. +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream.. The merging works by following a set of rules: * Merge into the top-level delta from the youngest ancestor delta to the oldest one @@ -72,10 +72,9 @@ The merging works by following a set of rules: * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. -Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. Here it is already possible to stream the result, which is feasible only if the memory of the base buffer + the memory of the TDS are smaller than a full size target buffer. Streaming will always make sense if the peak resulting from having the base, target and TDS buffers in memory together is unaffordable. -The memory consumption during the TDS processing are the uncompressed delta-bytes, the parsed DS, as well as the TDS. Afterwards one requires an allocated base buffer, the target buffer, as well as the TDS. -It is clearly visible that the current implementation does not at all reduce memory consumption, but the opposite is true as the TDS can be large for large files. +The memory consumption during the TDS processing is only the condensed delta-bytes, for each ADS an additional index is required which costs 8 byte per DC. When applying the TDS, one requires an allocated base buffer too.The target buffer can be allocated, but may be a writer as well. Performance Results ------------------- @@ -83,17 +82,16 @@ The benchmarking context was the same as for the brute-force GitDB algorithm. Th The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. -The throughput reaches 16.7 MiB/s, which equals 1344 streams/s, which makes it 15 times faster than the pure python version, and amazingly even 1.5 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. +The throughput reaches 15.2 MiB/s, which equals 1221 streams/s, which makes it nearly 14 times faster than the pure python version, and amazingly even 1.35 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. -All this comes at a relatively high memory consumption.Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. - -A 125 MB file took 3.1 seconds to unpack for instance, which is only 33% slower than the c implementation of the brute-force algorithm. +A 125 MB file took 2.5 seconds to unpack for instance, which is only 20% slower than the c implementation of the brute-force algorithm. Future work =========== -The current implementation of the reverse delta aggregation algorithm is already working well and fast, but leaves room for improvement in the realm of its memory consumption. One way to considerably reduce it would be to index the delta stream to determine bounds, instead of parsing it into a separate data structure Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. + +A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index f9169ffb0..32890dcb3 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -13,7 +13,7 @@ class TestPackedDBPerformance(TestBigRepoR): - def test_pack_random_access(self): + def _test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # sha lookup @@ -61,7 +61,7 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - def _disabled_test_correctness(self): + def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" From 2ddc5bad224d8f545ef3bb2ab3df98dfe063c5b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 12 Nov 2010 18:49:01 +0100 Subject: [PATCH 0119/1790] Updated to latest revision of async to support new thread-shutdown functionality --- ext/async | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/async b/ext/async index 5992bb6c8..7298742e1 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 5992bb6c85973ed81c54c71fef42e2413cd29e88 +Subproject commit 7298742e1b7236f2a4e369f915722a6618cef736 From 2a048f43d89112ff1f78ee05b59a9663e981f63f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 18 Nov 2010 23:42:14 +0100 Subject: [PATCH 0120/1790] Changed name/id of async submodule to something that doesn't look like a path --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 45ddc0b4c..9adc6121b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "ext/async"] +[submodule "async"] path = ext/async url = git://gitorious.org/git-python/async.git From 479ac8efc6aa6c579cba48bbb87f85f1cd0654f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Nov 2010 22:51:56 +0100 Subject: [PATCH 0121/1790] bumped version to 0.5.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e50e0fe7..6ebef0fa7 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.1", + version = "0.5.2", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", From dd4704095dbdc2770b9289b491bb166fa1d36a8d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Nov 2010 22:59:24 +0100 Subject: [PATCH 0122/1790] Updated changelog for 0.5.2 --- doc/source/changes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 84738ae05..7b8ebecc6 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -1,6 +1,12 @@ ######### Changelog ######### + +***** +0.5.2 +***** +* Improved performance of the c implementation, which now uses reverse-delta-aggregation to make a memory bound operation CPU bound. + ***** 0.5.1 ***** From 5d91a53372caa54cf7110cfa7fe1166956edc9c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 11:37:20 +0100 Subject: [PATCH 0123/1790] setup: added missing _delta_apply.c file to setup script, allowing the performance module to be compiled --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ebef0fa7..c7d0bd81c 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c'])], + ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 6edc28d6ab8b3f88673fd701b2c40104825a95df Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 11:54:00 +0100 Subject: [PATCH 0124/1790] Added delta_apply.h file to make more native use of python's build system, which should hopefully fix the easy_install trouble --- MANIFEST.in | 2 ++ _delta_apply.c | 4 +++- _delta_apply.h | 6 ++++++ _fun.c | 2 +- setup.py | 3 ++- 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 _delta_apply.h diff --git a/MANIFEST.in b/MANIFEST.in index a01acc452..7693cabf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,8 @@ include AUTHORS include README include _fun.c +include _delta_apply.c +include _delta_apply.h graft test diff --git a/_delta_apply.c b/_delta_apply.c index e99a803be..96ab30af9 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -1,10 +1,12 @@ -#include +#include "_delta_apply.h" #include #include #include #include #include + + typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; diff --git a/_delta_apply.h b/_delta_apply.h new file mode 100644 index 000000000..3e7e5f926 --- /dev/null +++ b/_delta_apply.h @@ -0,0 +1,6 @@ +#include + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams); +static PyObject* apply_delta(PyObject* self, PyObject* args); + +static PyTypeObject DeltaChunkListType; diff --git a/_fun.c b/_fun.c index befee4ec4..49970386f 100644 --- a/_fun.c +++ b/_fun.c @@ -1,5 +1,5 @@ #include -#include "_delta_apply.c" +#include "_delta_apply.h" static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { diff --git a/setup.py b/setup.py index c7d0bd81c..3a95ac845 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def run(self): except Exception: print "Ignored failure when building extensions, pure python modules will be used instead" # END ignore errors + def get_data_files(self): """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, @@ -77,7 +78,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'])], + ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 1bc281d31b8d31fd4dcbcd9b441b5c7b2c1b0bb5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 13:09:09 +0100 Subject: [PATCH 0125/1790] Added zip_safe flag to setup.py --- ext/async | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/async b/ext/async index 7298742e1..eccf3d63c 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 7298742e1b7236f2a4e369f915722a6618cef736 +Subproject commit eccf3d63c655e7a184ba339d747c98e965c1a63b diff --git a/setup.py b/setup.py index 3a95ac845..d235c91df 100755 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ def get_data_files(self): package_dir = {'gitdb':''}, ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], license = "BSD License", + zip_safe=False, requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', long_description = """GitDB is a pure-Python git object database""" From 9f977b8baaf9cbe9b38f3bdf4887cef5370b2229 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 25 Nov 2010 17:04:55 +0100 Subject: [PATCH 0126/1790] Switched async submodule to using github instead of gitorious --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9adc6121b..42efc2ed6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "async"] path = ext/async - url = git://gitorious.org/git-python/async.git + url = git://github.com/Byron/async.git From c0d5448fcd5ed6427ea96be01ef2d3ee509f3924 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:09:53 +0100 Subject: [PATCH 0127/1790] Ajusted all links to point to new repository on github --- README => README.rst | 5 ++--- doc/source/intro.rst | 5 ++--- ext/async | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) rename README => README.rst (84%) diff --git a/README b/README.rst similarity index 84% rename from README rename to README.rst index 33a566d86..753eb701c 100644 --- a/README +++ b/README.rst @@ -15,8 +15,7 @@ SOURCE ====== The source is available in a git repository at gitorious and github: -git://gitorious.org/git-python/gitdb.git -git://github.com/Byron/gitdb.git +git://github.com/gitpython-developers/gitdb.git Once the clone is complete, please be sure to initialize the submodules using @@ -33,7 +32,7 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -http://byronimo.lighthouseapp.com/projects/51787-gitpython +https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4d675cbc4..8fc0ec098 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -29,10 +29,9 @@ It is advised to have a look at the :ref:`Usage Guide ` for a br ================= Source Repository ================= -The latest source can be cloned using git from one of the following locations: +The latest source can be cloned using git from github: - * git://gitorious.org/git-python/gitdb.git - * git://github.com/Byron/gitdb.git + * git://github.com/gitpython-developers/gitdb.git License Information =================== diff --git a/ext/async b/ext/async index eccf3d63c..89790abd2 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit eccf3d63c655e7a184ba339d747c98e965c1a63b +Subproject commit 89790abd29bb4be851095b2f2c4c624b896b6e20 From 9fbc59da76b15cecb1ee37a8e48617fab58a077c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:24:07 +0100 Subject: [PATCH 0128/1790] moved all relevant files into the gitdb folder. Submodule relinked to point to new github location, and moved as well --- .gitmodules | 4 ++-- __init__.py => gitdb/__init__.py | 0 _delta_apply.c => gitdb/_delta_apply.c | 0 _delta_apply.h => gitdb/_delta_apply.h | 0 _fun.c => gitdb/_fun.c | 0 base.py => gitdb/base.py | 0 {db => gitdb/db}/__init__.py | 0 {db => gitdb/db}/base.py | 0 {db => gitdb/db}/git.py | 0 {db => gitdb/db}/loose.py | 0 {db => gitdb/db}/mem.py | 0 {db => gitdb/db}/pack.py | 0 {db => gitdb/db}/ref.py | 0 exc.py => gitdb/exc.py | 0 {ext => gitdb/ext}/async | 0 fun.py => gitdb/fun.py | 0 pack.py => gitdb/pack.py | 0 stream.py => gitdb/stream.py | 0 {test => gitdb/test}/__init__.py | 0 {test => gitdb/test}/db/__init__.py | 0 {test => gitdb/test}/db/lib.py | 0 {test => gitdb/test}/db/test_git.py | 0 {test => gitdb/test}/db/test_loose.py | 0 {test => gitdb/test}/db/test_mem.py | 0 {test => gitdb/test}/db/test_pack.py | 0 {test => gitdb/test}/db/test_ref.py | 0 .../7b/b839852ed5e3a069966281bb08d50012fb309b | Bin ...ack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx | Bin ...ck-11fdfa9e156ab73caae3b6da867192221f2089c2.pack | Bin ...ack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx | Bin ...ck-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack | Bin ...ack-c0438c19fb16422b6bbcce24387b3264416d485b.idx | Bin ...ck-c0438c19fb16422b6bbcce24387b3264416d485b.pack | Bin {test => gitdb/test}/lib.py | 0 {test => gitdb/test}/performance/lib.py | 0 {test => gitdb/test}/performance/test_pack.py | 0 .../test}/performance/test_pack_streaming.py | 0 {test => gitdb/test}/performance/test_stream.py | 0 {test => gitdb/test}/test_base.py | 0 {test => gitdb/test}/test_example.py | 0 {test => gitdb/test}/test_pack.py | 0 {test => gitdb/test}/test_stream.py | 0 {test => gitdb/test}/test_util.py | 0 typ.py => gitdb/typ.py | 0 util.py => gitdb/util.py | 0 45 files changed, 2 insertions(+), 2 deletions(-) rename __init__.py => gitdb/__init__.py (100%) rename _delta_apply.c => gitdb/_delta_apply.c (100%) rename _delta_apply.h => gitdb/_delta_apply.h (100%) rename _fun.c => gitdb/_fun.c (100%) rename base.py => gitdb/base.py (100%) rename {db => gitdb/db}/__init__.py (100%) rename {db => gitdb/db}/base.py (100%) rename {db => gitdb/db}/git.py (100%) rename {db => gitdb/db}/loose.py (100%) rename {db => gitdb/db}/mem.py (100%) rename {db => gitdb/db}/pack.py (100%) rename {db => gitdb/db}/ref.py (100%) rename exc.py => gitdb/exc.py (100%) rename {ext => gitdb/ext}/async (100%) rename fun.py => gitdb/fun.py (100%) rename pack.py => gitdb/pack.py (100%) rename stream.py => gitdb/stream.py (100%) rename {test => gitdb/test}/__init__.py (100%) rename {test => gitdb/test}/db/__init__.py (100%) rename {test => gitdb/test}/db/lib.py (100%) rename {test => gitdb/test}/db/test_git.py (100%) rename {test => gitdb/test}/db/test_loose.py (100%) rename {test => gitdb/test}/db/test_mem.py (100%) rename {test => gitdb/test}/db/test_pack.py (100%) rename {test => gitdb/test}/db/test_ref.py (100%) rename {test => gitdb/test}/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b (100%) rename {test => gitdb/test}/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack (100%) rename {test => gitdb/test}/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack (100%) rename {test => gitdb/test}/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack (100%) rename {test => gitdb/test}/lib.py (100%) rename {test => gitdb/test}/performance/lib.py (100%) rename {test => gitdb/test}/performance/test_pack.py (100%) rename {test => gitdb/test}/performance/test_pack_streaming.py (100%) rename {test => gitdb/test}/performance/test_stream.py (100%) rename {test => gitdb/test}/test_base.py (100%) rename {test => gitdb/test}/test_example.py (100%) rename {test => gitdb/test}/test_pack.py (100%) rename {test => gitdb/test}/test_stream.py (100%) rename {test => gitdb/test}/test_util.py (100%) rename typ.py => gitdb/typ.py (100%) rename util.py => gitdb/util.py (100%) diff --git a/.gitmodules b/.gitmodules index 42efc2ed6..9f06f7d07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "async"] - path = ext/async - url = git://github.com/Byron/async.git + path = gitdb/ext/async + url = git://github.com/gitpython-developers/async.git diff --git a/__init__.py b/gitdb/__init__.py similarity index 100% rename from __init__.py rename to gitdb/__init__.py diff --git a/_delta_apply.c b/gitdb/_delta_apply.c similarity index 100% rename from _delta_apply.c rename to gitdb/_delta_apply.c diff --git a/_delta_apply.h b/gitdb/_delta_apply.h similarity index 100% rename from _delta_apply.h rename to gitdb/_delta_apply.h diff --git a/_fun.c b/gitdb/_fun.c similarity index 100% rename from _fun.c rename to gitdb/_fun.c diff --git a/base.py b/gitdb/base.py similarity index 100% rename from base.py rename to gitdb/base.py diff --git a/db/__init__.py b/gitdb/db/__init__.py similarity index 100% rename from db/__init__.py rename to gitdb/db/__init__.py diff --git a/db/base.py b/gitdb/db/base.py similarity index 100% rename from db/base.py rename to gitdb/db/base.py diff --git a/db/git.py b/gitdb/db/git.py similarity index 100% rename from db/git.py rename to gitdb/db/git.py diff --git a/db/loose.py b/gitdb/db/loose.py similarity index 100% rename from db/loose.py rename to gitdb/db/loose.py diff --git a/db/mem.py b/gitdb/db/mem.py similarity index 100% rename from db/mem.py rename to gitdb/db/mem.py diff --git a/db/pack.py b/gitdb/db/pack.py similarity index 100% rename from db/pack.py rename to gitdb/db/pack.py diff --git a/db/ref.py b/gitdb/db/ref.py similarity index 100% rename from db/ref.py rename to gitdb/db/ref.py diff --git a/exc.py b/gitdb/exc.py similarity index 100% rename from exc.py rename to gitdb/exc.py diff --git a/ext/async b/gitdb/ext/async similarity index 100% rename from ext/async rename to gitdb/ext/async diff --git a/fun.py b/gitdb/fun.py similarity index 100% rename from fun.py rename to gitdb/fun.py diff --git a/pack.py b/gitdb/pack.py similarity index 100% rename from pack.py rename to gitdb/pack.py diff --git a/stream.py b/gitdb/stream.py similarity index 100% rename from stream.py rename to gitdb/stream.py diff --git a/test/__init__.py b/gitdb/test/__init__.py similarity index 100% rename from test/__init__.py rename to gitdb/test/__init__.py diff --git a/test/db/__init__.py b/gitdb/test/db/__init__.py similarity index 100% rename from test/db/__init__.py rename to gitdb/test/db/__init__.py diff --git a/test/db/lib.py b/gitdb/test/db/lib.py similarity index 100% rename from test/db/lib.py rename to gitdb/test/db/lib.py diff --git a/test/db/test_git.py b/gitdb/test/db/test_git.py similarity index 100% rename from test/db/test_git.py rename to gitdb/test/db/test_git.py diff --git a/test/db/test_loose.py b/gitdb/test/db/test_loose.py similarity index 100% rename from test/db/test_loose.py rename to gitdb/test/db/test_loose.py diff --git a/test/db/test_mem.py b/gitdb/test/db/test_mem.py similarity index 100% rename from test/db/test_mem.py rename to gitdb/test/db/test_mem.py diff --git a/test/db/test_pack.py b/gitdb/test/db/test_pack.py similarity index 100% rename from test/db/test_pack.py rename to gitdb/test/db/test_pack.py diff --git a/test/db/test_ref.py b/gitdb/test/db/test_ref.py similarity index 100% rename from test/db/test_ref.py rename to gitdb/test/db/test_ref.py diff --git a/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b similarity index 100% rename from test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b rename to gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx similarity index 100% rename from test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx rename to gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack b/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack similarity index 100% rename from test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack rename to gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx similarity index 100% rename from test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx rename to gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack b/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack similarity index 100% rename from test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack rename to gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx similarity index 100% rename from test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx rename to gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack similarity index 100% rename from test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack rename to gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack diff --git a/test/lib.py b/gitdb/test/lib.py similarity index 100% rename from test/lib.py rename to gitdb/test/lib.py diff --git a/test/performance/lib.py b/gitdb/test/performance/lib.py similarity index 100% rename from test/performance/lib.py rename to gitdb/test/performance/lib.py diff --git a/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py similarity index 100% rename from test/performance/test_pack.py rename to gitdb/test/performance/test_pack.py diff --git a/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py similarity index 100% rename from test/performance/test_pack_streaming.py rename to gitdb/test/performance/test_pack_streaming.py diff --git a/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py similarity index 100% rename from test/performance/test_stream.py rename to gitdb/test/performance/test_stream.py diff --git a/test/test_base.py b/gitdb/test/test_base.py similarity index 100% rename from test/test_base.py rename to gitdb/test/test_base.py diff --git a/test/test_example.py b/gitdb/test/test_example.py similarity index 100% rename from test/test_example.py rename to gitdb/test/test_example.py diff --git a/test/test_pack.py b/gitdb/test/test_pack.py similarity index 100% rename from test/test_pack.py rename to gitdb/test/test_pack.py diff --git a/test/test_stream.py b/gitdb/test/test_stream.py similarity index 100% rename from test/test_stream.py rename to gitdb/test/test_stream.py diff --git a/test/test_util.py b/gitdb/test/test_util.py similarity index 100% rename from test/test_util.py rename to gitdb/test/test_util.py diff --git a/typ.py b/gitdb/typ.py similarity index 100% rename from typ.py rename to gitdb/typ.py diff --git a/util.py b/gitdb/util.py similarity index 100% rename from util.py rename to gitdb/util.py From 0300d0d6c4af63257d094813ceaab2302906680c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:45:49 +0100 Subject: [PATCH 0129/1790] Fixed unittests --- doc/source/tutorial.rst | 2 +- gitdb/__init__.py | 8 +++++++- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_pack.py | 3 +-- gitdb/test/db/test_ref.py | 2 +- gitdb/test/test_example.py | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index cfe3fb284..55a737f6b 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -37,7 +37,7 @@ Both have two sets of methods, one of which allows interacting with single objec Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: - ldb = LooseObjectDB(fixture_path("../../.git/objects")) + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index d79788fc4..c8e77759e 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -6,7 +6,13 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext')) + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async')) + + try: + import async + except ImportError: + raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH") + #END verify import #} END initialization diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index d2ae10bad..bcab1fc55 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,7 +7,7 @@ class TestGitDB(TestDBBase): def test_reading(self): - gdb = GitDB(fixture_path('../../.git/objects')) + gdb = GitDB(fixture_path('../../../.git/objects')) # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 0386b3f80..1d0cb96e7 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -15,8 +15,7 @@ def test_writing(self, path): pdb = PackedDB(path) # on demand, we init our pack cache - num_packs = 2 - assert len(pdb.entities()) == num_packs + num_packs = len(pdb.entities()) assert pdb._st_mtime != 0 # test pack directory changed: diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 9df25cef6..af75016b0 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -34,7 +34,7 @@ def test_writing(self, path): # setup alternate file # add two, one is invalid - own_repo_path = fixture_path('../../.git/objects') # use own repo + own_repo_path = fixture_path('../../../.git/objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 3fc3fcf5e..2d6096132 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -11,7 +11,7 @@ class TestExamples(TestBase): def test_base(self): - ldb = LooseObjectDB(fixture_path("../../.git/objects")) + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) From 88d500edf2163be3b249ae288a06b725934560d9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:57:16 +0100 Subject: [PATCH 0130/1790] setup and doc generation works once again --- MANIFEST.in | 8 ++++---- doc/source/conf.py | 2 +- gitdb/fun.py | 9 +++++---- gitdb/pack.py | 2 +- setup.py | 7 +++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7693cabf8..b14aed9b3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,11 +4,11 @@ include CHANGES include AUTHORS include README -include _fun.c -include _delta_apply.c -include _delta_apply.h +include gitdb/_fun.c +include gitdb/_delta_apply.c +include gitdb/_delta_apply.h -graft test +prune gitdb/test global-exclude .git* global-exclude *.pyc diff --git a/doc/source/conf.py b/doc/source/conf.py index e10addb8d..28deb3106 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('../../../')) +sys.path.append(os.path.abspath('../../')) # -- General configuration ----------------------------------------------------- diff --git a/gitdb/fun.py b/gitdb/fun.py index 0b14f82ac..fc4172040 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -44,8 +44,7 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'reverse_connect_deltas', - 'connect_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList') #{ Structures @@ -492,8 +491,10 @@ def stream_copy(read, write, size, chunk_size): return dbw def connect_deltas(dstreams): - """Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" diff --git a/gitdb/pack.py b/gitdb/pack.py index 30da52c63..affcbe275 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -651,7 +651,7 @@ def is_valid_stream(self, sha, use_crc=False): :param use_crc: if True, the index' crc for the sha is used to determine :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is + whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the data of the actual undeltified object, as it depends on more than just this stream. diff --git a/setup.py b/setup.py index d235c91df..3c6617422 100755 --- a/setup.py +++ b/setup.py @@ -75,10 +75,9 @@ def get_data_files(self): author_email = "byronimo@gmail.com", url = "http://gitorious.org/git-python/gitdb", packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), - package_data={'gitdb' : ['AUTHORS', 'README'], - 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, - package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], + package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + package_dir = {'gitdb':'gitdb'}, + ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, requires=('async (>=0.6.1)',), From 1fe2a9403fafa810af25062c7e6b20be8e6be480 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 1 Dec 2010 10:28:14 +0100 Subject: [PATCH 0131/1790] setup .gitmodules to use a trackin branch automatically --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 9f06f7d07..3db4c676d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "async"] path = gitdb/ext/async url = git://github.com/gitpython-developers/async.git + branch = master From 6d315b8a92ae2cba936bd38e433592383f42cc10 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 23 Feb 2011 00:26:14 +0100 Subject: [PATCH 0132/1790] Added license information file --- LICENSE | 30 ++++++++++++++++++++++++++++++ gitdb/ext/async | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..be11e73c1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the GitDB project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/gitdb/ext/async b/gitdb/ext/async index 89790abd2..e91b46928 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 89790abd29bb4be851095b2f2c4c624b896b6e20 +Subproject commit e91b4692825e0121504262ba58ac0b2bcd09fea3 From df570f00f611073a20796128ca167474aa7826fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 23 Feb 2011 00:43:31 +0100 Subject: [PATCH 0133/1790] preprended all modules with licensing information --- gitdb/__init__.py | 4 ++++ gitdb/base.py | 4 ++++ gitdb/db/__init__.py | 4 ++++ gitdb/db/base.py | 4 ++++ gitdb/db/git.py | 4 ++++ gitdb/db/loose.py | 4 ++++ gitdb/db/mem.py | 4 ++++ gitdb/db/pack.py | 4 ++++ gitdb/db/ref.py | 4 ++++ gitdb/exc.py | 4 ++++ gitdb/ext/async | 2 +- gitdb/fun.py | 4 ++++ gitdb/pack.py | 4 ++++ gitdb/stream.py | 4 ++++ gitdb/test/__init__.py | 4 ++++ gitdb/test/db/__init__.py | 4 ++++ gitdb/test/db/lib.py | 4 ++++ gitdb/test/db/test_git.py | 4 ++++ gitdb/test/db/test_loose.py | 4 ++++ gitdb/test/db/test_mem.py | 4 ++++ gitdb/test/db/test_pack.py | 4 ++++ gitdb/test/db/test_ref.py | 4 ++++ gitdb/test/lib.py | 4 ++++ gitdb/test/performance/lib.py | 4 ++++ gitdb/test/performance/test_pack.py | 4 ++++ gitdb/test/performance/test_pack_streaming.py | 4 ++++ gitdb/test/performance/test_stream.py | 4 ++++ gitdb/test/test_base.py | 4 ++++ gitdb/test/test_example.py | 4 ++++ gitdb/test/test_pack.py | 4 ++++ gitdb/test/test_stream.py | 4 ++++ gitdb/test/test_util.py | 4 ++++ gitdb/typ.py | 4 ++++ gitdb/util.py | 4 ++++ 34 files changed, 133 insertions(+), 1 deletion(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c8e77759e..a551f37dc 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Initialize the object database module""" import sys diff --git a/gitdb/base.py b/gitdb/base.py index d0bcc0866..ff1062bf6 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( bin_to_hex, diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index 85a0a6874..e5935b7c2 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import * from loose import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 1914dbbce..2189d4193 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( pool, diff --git a/gitdb/db/git.py b/gitdb/db/git.py index f0e63b15a..b8fc46aa0 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( CompoundDB, ObjectDBW, diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 521be44c2..6cd1cefd5 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( FileDBBase, ObjectDBR, diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index f361ab801..8012ad15e 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains the MemoryDatabase implementation""" from loose import LooseObjectDB from base import ( diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 0ec8a4e3b..eef3f712e 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" from base import ( FileDBBase, diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index c149c03d0..898984323 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( CompoundDB, ) diff --git a/gitdb/exc.py b/gitdb/exc.py index 012cdbc6b..96fa874e3 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with common exceptions""" from util import to_hex_sha diff --git a/gitdb/ext/async b/gitdb/ext/async index e91b46928..10310824c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit e91b4692825e0121504262ba58ac0b2bcd09fea3 +Subproject commit 10310824c001deab8fea85b88ebda0696f964b3e diff --git a/gitdb/fun.py b/gitdb/fun.py index fc4172040..3e035dbf1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains basic c-functions which usually contain performance critical code Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" diff --git a/gitdb/pack.py b/gitdb/pack.py index affcbe275..09e7defc5 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( BadObject, diff --git a/gitdb/stream.py b/gitdb/stream.py index 0d8972898..6c3b8d31f 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from cStringIO import StringIO import errno diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 0dec7750f..760f531be 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php import gitdb.util diff --git a/gitdb/test/db/__init__.py b/gitdb/test/db/__init__.py index e69de29bb..8a681e428 100644 --- a/gitdb/test/db/__init__.py +++ b/gitdb/test/db/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 0080d919e..416c8c588 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index bcab1fc55..310116351 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.exc import BadObject from gitdb.db import GitDB diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 8e8a9bfc3..ee2d78d08 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import LooseObjectDB from gitdb.exc import BadObject diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 4a9b7ee12..188cb0a93 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( MemoryDB, diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 1d0cb96e7..e8ba6f8fc 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import PackedDB from gitdb.test.lib import fixture_path diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index af75016b0..0d8eeebb3 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ReferenceDB diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 3fb87d547..342234adc 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( OStream, diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 45e0ca53f..761113d51 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os from gitdb.test.lib import * diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 32890dcb3..da952b17a 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" from lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 4d47cdfcc..22a62a39d 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" from lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 1afc1a1a0..f5f2e2e4d 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" from lib import TestBigRepoR from gitdb.db import * diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 740e50bcd..1b20faf87 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( TestBase, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 2d6096132..753177560 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" from lib import * from gitdb import IStream diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 770a78bad..928f0cd97 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" from lib import ( TestBase, diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 948cbe766..523f77056 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( TestBase, diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 6a389d27c..90f4156b9 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" import tempfile import os diff --git a/gitdb/typ.py b/gitdb/typ.py index 54a1f84be..e84dd2455 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" #{ String types diff --git a/gitdb/util.py b/gitdb/util.py index 1ea182025..4bb3c7352 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php import binascii import os import mmap From 3bcb30f1916239f1af25b4d6d8934c64bb47f8ea Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 11:06:36 +0200 Subject: [PATCH 0134/1790] Added qt creator project as it has advantages regarding the navigation over jEdit, although jedit has advantages regarding the syntax highlighting and whitespace visualization --- gitdb.pro | 48 +++++++++ gitdb.pro.user | 267 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 gitdb.pro create mode 100644 gitdb.pro.user diff --git a/gitdb.pro b/gitdb.pro new file mode 100644 index 000000000..a682615c4 --- /dev/null +++ b/gitdb.pro @@ -0,0 +1,48 @@ + +OTHER_FILES += \ + setup.py \ + README.rst \ + MANIFEST \ + Makefile \ + LICENSE \ + AUTHORS \ + gitdb/util.py \ + gitdb/typ.py \ + gitdb/stream.py \ + gitdb/pack.py \ + gitdb/__init__.py \ + gitdb/fun.py \ + gitdb/exc.py \ + gitdb/base.py \ + doc/source/tutorial.rst \ + doc/source/intro.rst \ + doc/source/index.rst \ + doc/source/conf.py \ + doc/source/changes.rst \ + doc/source/api.rst \ + doc/source/algorithm.rst \ + gitdb/db/ref.py \ + gitdb/db/pack.py \ + gitdb/db/mem.py \ + gitdb/db/loose.py \ + gitdb/db/__init__.py \ + gitdb/db/git.py \ + gitdb/db/base.py \ + gitdb/test/test_util.py \ + gitdb/test/test_stream.py \ + gitdb/test/test_pack.py \ + gitdb/test/test_example.py \ + gitdb/test/test_base.py \ + gitdb/test/lib.py \ + gitdb/test/__init__.py \ + gitdb/test/performance/test_stream.py \ + gitdb/test/performance/test_pack_streaming.py \ + gitdb/test/performance/test_pack.py \ + gitdb/test/performance/lib.py + +HEADERS += \ + gitdb/_delta_apply.h + +SOURCES += \ + gitdb/_fun.c \ + gitdb/_delta_apply.c diff --git a/gitdb.pro.user b/gitdb.pro.user new file mode 100644 index 000000000..398cb70a1 --- /dev/null +++ b/gitdb.pro.user @@ -0,0 +1,267 @@ + + + + ProjectExplorer.Project.ActiveTarget + 0 + + + ProjectExplorer.Project.EditorSettings + + Default + + + + ProjectExplorer.Project.Target.0 + + Desktop + + Qt4ProjectManager.Target.DesktopTarget + 0 + 0 + 1 + + + 0 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + false + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + 4 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + No deployment + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + gitdb + + Qt4ProjectManager.Qt4RunConfiguration + 2 + + gitdb.pro + false + false + + false + + 3768 + true + false + + + + /usr/bin/nosetests + -s + gitdb/test/test_pack.py + + 2 + python + false + + $BUILDDIR + Run python + test-pack + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + true + false + + 2 + + + + ProjectExplorer.Project.TargetCount + 1 + + + ProjectExplorer.Project.Updater.EnvironmentId + {d38778e3-6b24-4419-8fc3-c8cc320f55e0} + + + ProjectExplorer.Project.Updater.FileVersion + 8 + + From 810d1e38315c6e886c1daef93670840b213ee78a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 11:08:26 +0200 Subject: [PATCH 0135/1790] Added stub for pack writing implementation which should work for pack streaming over a transport as well --- gitdb/fun.py | 10 +--------- gitdb/pack.py | 25 +++++++++++++++++++------ gitdb/test/test_pack.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 3e035dbf1..34978cd97 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -411,15 +411,7 @@ def pack_object_header_info(data): size += (c & 0x7f) << s s += 7 # END character loop - - try: - return (type_id, size, i) - except KeyError: - # invalid object type - we could try to be smart now and decode part - # of the stream to get the info, problem is that we had trouble finding - # the exact start of the content stream - raise BadObjectType(type_id) - # END handle exceptions + return (type_id, size, i) def msb_size(data, offset=0): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index 09e7defc5..335fe3c00 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -98,8 +98,7 @@ def pack_object_at(data, offset, as_stream): # REF DELTA elif type_id == REF_DELTA: total_rela_offset = data_rela_offset+20 - ref_sha = data[data_rela_offset:total_rela_offset] - delta_info = ref_sha + delta_info = data[data_rela_offset:total_rela_offset] # BASE OBJECT else: # assume its a base object @@ -561,11 +560,10 @@ def _sha_to_index(self, sha): def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" - indexfile = self._index + _sha = self._index.sha _object = self._object - for index in xrange(indexfile.size()): - sha = indexfile.sha(index) - yield _object(sha, as_stream, index) + for index in xrange(self._index.size()): + yield _object(_sha(index), as_stream, index) # END for each index def _object(self, sha, as_stream, index=-1): @@ -760,5 +758,20 @@ def collect_streams(self, sha): return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + @classmethod + def create(cls, object_iter, pack_write, index_write=None): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :note: The destination of the write functions is up to the user. It could + be a socket, or a file for instance""" + + #} END interface diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 928f0cd97..8e98808c7 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -25,8 +25,12 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from itertools import izip +from itertools import izip, chain +from nose import SkipTest + import os +import sys +import tempfile #{ Utilities @@ -134,7 +138,9 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def test_pack_entity(self): + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_iterators = list(); for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): @@ -143,6 +149,7 @@ def test_pack_entity(self): entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile + pack_iterators.append(entity.stream_iter()) count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): @@ -174,9 +181,28 @@ def test_pack_entity(self): # END for each info, stream tuple assert count == size - # END for each entity + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path = tempfile.mktemp('', "pack", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + for pp, ip in ((pack_path, )*2, (index_path, None)): + pfile = open(pp, 'wb') + ifile = None + if ip: + ifile = open(ip, 'wb') + #END handle ip + + PackEntity.create(chain(*pack_iterators), pfile, ifile) + assert os.path.getsize(pp) > 100 + if ip is not None: + assert os.path.getsize(ip) > 100 + #END verify files exist + #END for each packpath, indexpath pair + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack - pass + raise SkipTest() From e83210d99aaac5768827c448909fa04d63776e64 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 18:01:05 +0200 Subject: [PATCH 0136/1790] initial version of pack writing, which seems to work, but still needs some more testing and verification --- gitdb/exc.py | 3 + gitdb/fun.py | 20 +++- gitdb/pack.py | 196 ++++++++++++++++++++++++++++++++++++++-- gitdb/stream.py | 18 +++- gitdb/test/test_pack.py | 51 ++++++++--- 5 files changed, 266 insertions(+), 22 deletions(-) diff --git a/gitdb/exc.py b/gitdb/exc.py index 96fa874e3..e087047b4 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -17,6 +17,9 @@ class BadObject(ODBError): def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) + +class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): """Thrown if a possibly shortened name does not uniquely represent a single object diff --git a/gitdb/fun.py b/gitdb/fun.py index 34978cd97..5bbe8efc3 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -48,7 +48,7 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures @@ -412,6 +412,24 @@ def pack_object_header_info(data): s += 7 # END character loop return (type_id, size, i) + +def create_pack_object_header(obj_type, obj_size): + """:return: string defining the pack header comprised of the object type + and its incompressed size in bytes + :parmam obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = str() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + return hdr def msb_size(data, offset=0): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index 335fe3c00..6c32949d6 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -5,7 +5,8 @@ """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( BadObject, - UnsupportedOperation + UnsupportedOperation, + ParseError ) from util import ( zlib, @@ -15,6 +16,7 @@ ) from fun import ( + create_pack_object_header, pack_object_header_info, is_equal_canonical_sha, type_id_to_type_map, @@ -47,6 +49,7 @@ DeltaApplyReader, Sha1Writer, NullStream, + FlexibleSha1Writer ) from struct import ( @@ -54,6 +57,8 @@ unpack, ) +from binascii import crc32 + from itertools import izip import array import os @@ -119,10 +124,113 @@ def pack_object_at(data, offset, as_stream): return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) # END handle info # END handle stream - + +def write_stream_to_pack(read, write, zstream, want_crc=False): + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param want_crc: if True, the crc will be generated over the compressed data. + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if want_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + crc = 0 + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + if len(chunk) != chunk_size: + break + #END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + return (br, bw, crc) + + #} END utilities +class IndexWriter(object): + """Utility to cache index information, allowing to write all information later + in one go to the given stream + :note: currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_binsha, write): + """Write the index file using the given write method + :param pack_binsha: sha over the whole pack that we index""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,)*256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[ord(t[0][0])] += 1 + #END prepare fanout + + for i in xrange(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i+1] = v + #END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1]&0xffffffff)) + #END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist)-1 + #END hande 64 bit offsets + sha_write(pack('>L', ofs&0xffffffff)) + #END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + #END for each offset + + # trailer + assert(len(pack_binsha) == 20) + sha_write(pack_binsha) + write(sha_writer.sha(as_hex=False)) + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find @@ -135,6 +243,8 @@ class PackIndexFile(LazyMixin): # used in v2 indices _sha_list_offset = 8 + 1024 + index_v2_signature = '\377tOc' + index_version_default = 2 def __init__(self, indexpath): super(PackIndexFile, self).__init__() @@ -155,7 +265,7 @@ def _set_cache_(self, attr): # to access the fanout table or related properties # CHECK VERSION - self._version = (self._data[:4] == '\377tOc' and 2) or 1 + self._version = (self._data[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: version_id = unpack_from(">L", self._data, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id @@ -383,6 +493,8 @@ class PackFile(LazyMixin): case""" __slots__ = ('_packpath', '_data', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 # offset into our data at which the first object starts first_object_offset = 3*4 # header bytes @@ -396,15 +508,19 @@ def _set_cache_(self, attr): self._data = file_contents_ro_filepath(self._packpath) # read the header information - type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) + type_id, self._version, self._size = unpack_from(">LLL", self._data, 0) # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? else: # must be '_size' or '_version' # read header info - we do that just with a file stream - type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12)) + type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12)) # END handle header + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + #END assert type id + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data @@ -759,7 +875,8 @@ def collect_streams(self, sha): @classmethod - def create(cls, object_iter, pack_write, index_write=None): + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. @@ -769,9 +886,74 @@ def create(cls, object_iter, pack_write, index_write=None): :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: binary sha over all the contents of the pack :note: The destination of the write functions is up to the user. It could - be a socket, or a file for instance""" + be a socket, or a file for instance + :note: writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + #END handle list type + object_count = len(objs) + #END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + #END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, want_crc = index_write) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + #END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + #END abort once we are done + #END for each object + + if actual_count != object_count: + raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + #END count assertion + + # write footer + binsha = pack_writer.sha(as_hex = False) + assert len(binsha) == 20 + pack_write(binsha) + ofs += len(binsha) # just for completeness ;) + + if wants_index: + index.write(binsha, index_write) + #END handle index + return binsha #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 6c3b8d31f..8010a0551 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,9 @@ except ImportError: pass -__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams @@ -557,6 +559,20 @@ def sha(self, as_hex = False): #} END interface +class FlexibleSha1Writer(Sha1Writer): + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) + + class ZippedStoreShaWriter(Sha1Writer): """Remembers everything someone writes to it and generates a sha""" __slots__ = ('buf', 'zip') diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 8e98808c7..c4d8df165 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -140,7 +140,7 @@ def test_pack(self): @with_rw_directory def test_pack_entity(self, rw_dir): - pack_iterators = list(); + pack_objs = list() for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): @@ -149,7 +149,7 @@ def test_pack_entity(self, rw_dir): entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile - pack_iterators.append(entity.stream_iter()) + pack_objs.extend(entity.stream_iter()) count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): @@ -182,24 +182,49 @@ def test_pack_entity(self, rw_dir): assert count == size # END for each entity - + # pack writing - write all packs into one # index path can be None pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) - for pp, ip in ((pack_path, )*2, (index_path, None)): - pfile = open(pp, 'wb') - ifile = None - if ip: - ifile = open(ip, 'wb') + iteration = 0 + for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + pfile = open(ppath, 'wb') + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write #END handle ip - PackEntity.create(chain(*pack_iterators), pfile, ifile) - assert os.path.getsize(pp) > 100 - if ip is not None: - assert os.path.getsize(ip) > 100 + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + for obj in pack_objs: + obj.stream.seek(0) + #END rewind streams + iteration += 1 + + binsha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pfile.close() + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == binsha + + # verify index + if ipath is not None: + assert os.path.getsize(ipath) > 100 + #END verify files exist - #END for each packpath, indexpath pair + + if ifile: + ifile.close() + #END handle index + #END for each packpath, indexpath pair + + # def test_pack_64(self): From 98a19ac1986b623277098263f01696827567c584 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 18:49:22 +0200 Subject: [PATCH 0137/1790] Implemented remainder of the test, and it already shows that something is wrong with my packs. Probably something stupid ;) --- gitdb/pack.py | 62 +++++++++++++++++++++++++++++++---------- gitdb/test/lib.py | 7 +++-- gitdb/test/test_pack.py | 34 +++++++++++++++------- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 6c32949d6..d90eeb991 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -12,6 +12,7 @@ zlib, LazyMixin, unpack_from, + bin_to_hex, file_contents_ro_filepath, ) @@ -60,6 +61,7 @@ from binascii import crc32 from itertools import izip +import tempfile import array import os import sys @@ -176,9 +178,10 @@ def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - def write(self, pack_binsha, write): + def write(self, pack_sha, write): """Write the index file using the given write method - :param pack_binsha: sha over the whole pack that we index""" + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) @@ -192,11 +195,10 @@ def write(self, pack_binsha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) - tmplist[i+1] = v + tmplist[i+1] += v #END write each fanout entry sha_write(pack('>L', tmplist[255])) @@ -226,9 +228,11 @@ def write(self, pack_binsha, write): #END for each offset # trailer - assert(len(pack_binsha) == 20) - sha_write(pack_binsha) - write(sha_writer.sha(as_hex=False)) + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha @@ -767,7 +771,9 @@ def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - :param use_crc: if True, the index' crc for the sha is used to determine + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. :param sha: 20 byte sha1 of the object whose stream to verify whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the @@ -890,7 +896,8 @@ def write_pack(cls, object_iter, pack_write, index_write=None, this would be the place to put it. Otherwise we have to pre-iterate and store all items into a list to get the number, which uses more memory than necessary. :param zlib_compression: the zlib compression level to use - :return: binary sha over all the contents of the pack + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None :note: The destination of the write functions is up to the user. It could be a socket, or a file for instance :note: writes only undeltified objects""" @@ -944,16 +951,41 @@ def write_pack(cls, object_iter, pack_write, index_write=None, #END count assertion # write footer - binsha = pack_writer.sha(as_hex = False) - assert len(binsha) == 20 - pack_write(binsha) - ofs += len(binsha) # just for completeness ;) + pack_sha = pack_writer.sha(as_hex = False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + index_sha = None if wants_index: - index.write(binsha, index_write) + index_sha = index.write(pack_sha, index_write) #END handle index - return binsha + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + :note: for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) #} END interface diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 342234adc..50645be65 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -42,19 +42,22 @@ def with_rw_directory(func): def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) + keep = False try: try: return func(self, path) except Exception: print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + keep = True raise finally: # Need to collect here to be sure all handles have been closed. It appears # a windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. - gc.collect() - shutil.rmtree(path) + if not keep: + gc.collect() + shutil.rmtree(path) # END handle exception # END wrapper diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index c4d8df165..e9c933e15 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -188,6 +188,10 @@ def test_pack_entity(self, rw_dir): pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + #END utility for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): pfile = open(ppath, 'wb') iwrite = None @@ -198,12 +202,11 @@ def test_pack_entity(self, rw_dir): # make sure we rewind the streams ... we work on the same objects over and over again if iteration > 0: - for obj in pack_objs: - obj.stream.seek(0) + rewind_streams() #END rewind streams iteration += 1 - binsha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) pfile.close() assert os.path.getsize(ppath) > 100 @@ -211,20 +214,31 @@ def test_pack_entity(self, rw_dir): pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default - assert pf.checksum() == binsha + assert pf.checksum() == pack_sha # verify index if ipath is not None: + ifile.close() assert os.path.getsize(ipath) > 100 - + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) #END verify files exist - - if ifile: - ifile.close() - #END handle index #END for each packpath, indexpath pair - # + # verify the packs throughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in reversed(range(2)): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + #END for each info + assert count == len(pack_objs) def test_pack_64(self): From 184a776960efdc2a83eac571c9c046ffcee3e7c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 20:27:44 +0200 Subject: [PATCH 0138/1790] crc needs to be done on the pack object header as well, of course --- gitdb/pack.py | 22 ++++++++++++++++++---- gitdb/test/test_pack.py | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index d90eeb991..7ae9786e6 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -127,15 +127,20 @@ def pack_object_at(data, offset, as_stream): # END handle info # END handle stream -def write_stream_to_pack(read, write, zstream, want_crc=False): +def write_stream_to_pack(read, write, zstream, base_crc=None): """Copy a stream as read from read function, zip it, and write the result. Count the number of written bytes and return it - :param want_crc: if True, the crc will be generated over the compressed data. - :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if want_crc + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc was false""" br = 0 # bytes read bw = 0 # bytes written + want_crc = base_crc is not None crc = 0 + if want_crc: + crc = base_crc + #END initialize crc while True: chunk = read(chunk_size) @@ -651,6 +656,9 @@ def __init__(self, pack_or_index_path): def _set_cache_(self, attr): # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" @@ -926,15 +934,21 @@ def write_pack(cls, object_iter, pack_write, index_write=None, actual_count = 0 for obj in objs: actual_count += 1 + crc = 0 # object header hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + #END handle crc pwrite(hdr) # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, want_crc = index_write) + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) assert(br == obj.size) if wants_index: index.append(obj.binsha, crc, ofs) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index e9c933e15..4a7f1caf2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -234,7 +234,7 @@ def rewind_streams(): count = 0 for info in entity.info_iter(): count += 1 - for use_crc in reversed(range(2)): + for use_crc in range(2): assert entity.is_valid_stream(info.binsha, use_crc) # END for each crc mode #END for each info From 0c7a3ec9829caa6632afd3e46901be67c63ae7fa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 23:40:04 +0200 Subject: [PATCH 0139/1790] Fixed _perf module, which built, but didn't link dynamically. All the time, I think it never successfully imported, but its hard to believe this slipped by. Added performance test for pack-writing, which isn't really showing what I want as it currently read data from a densly compressed pack which takes most of the time in the nearly pure python implementation. Compared to c++, all the measured performance is just below anything I'd want to use. But we shouldn't forget this is just a test implementation, writing packs is quite simple actually, if you leave out the delta compression part and the delta logic --- gitdb/_delta_apply.c | 19 ++++---- gitdb/_delta_apply.h | 6 +-- gitdb/test/performance/test_pack_streaming.py | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index 96ab30af9..f03e7ea6d 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -1,4 +1,4 @@ -#include "_delta_apply.h" +#include <_delta_apply.h> #include #include #include @@ -463,7 +463,7 @@ void DIV_reset(DeltaInfoVector* vec) // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! -static inline +inline DeltaInfo* DIV_append(DeltaInfoVector* vec) { if (vec->size + 1 > vec->reserved_size){ @@ -703,7 +703,7 @@ typedef struct { } DeltaChunkList; -static + int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) { if(args && PySequence_Size(args) > 0){ @@ -715,20 +715,20 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return 0; } -static + void DCL_dealloc(DeltaChunkList* self) { TSI_destroy(&(self->istream)); } -static + PyObject* DCL_py_rbound(DeltaChunkList* self) { return PyLong_FromUnsignedLongLong(self->istream.target_size); } // Write using a write function, taking remaining bytes from a base buffer -static + PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { PyObject* pybuf = 0; @@ -769,13 +769,13 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) Py_RETURN_NONE; } -static PyMethodDef DCL_methods[] = { +PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; -static PyTypeObject DeltaChunkListType = { +PyTypeObject DeltaChunkListType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "DeltaChunkList", /*tp_name*/ @@ -897,7 +897,7 @@ uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) return num_chunks; } -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator PyObject* stream_iter = 0; @@ -1088,7 +1088,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // Write using a write function, taking remaining bytes from a base buffer // replaces the corresponding method in python -static PyObject* apply_delta(PyObject* self, PyObject* args) { PyObject* pybbuf = 0; diff --git a/gitdb/_delta_apply.h b/gitdb/_delta_apply.h index 3e7e5f926..1fcd53832 100644 --- a/gitdb/_delta_apply.h +++ b/gitdb/_delta_apply.h @@ -1,6 +1,6 @@ #include -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams); -static PyObject* apply_delta(PyObject* self, PyObject* args); +extern PyObject* connect_deltas(PyObject *self, PyObject *dstreams); +extern PyObject* apply_delta(PyObject* self, PyObject* args); -static PyTypeObject DeltaChunkListType; +extern PyTypeObject DeltaChunkListType; diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 22a62a39d..795ed1e26 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -8,14 +8,57 @@ ) from gitdb.db.pack import PackedDB +from gitdb.stream import NullStream +from gitdb.pack import PackEntity import os import sys from time import time +from nose import SkipTest + +class CountedNullStream(NullStream): + __slots__ = '_bw' + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + class TestPackStreamingPerformance(TestBigRepoR): + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 5000 + count = 0 + total_size = 0 + st = time() + objs = list() + for sha in pdb.sha_iter(): + count += 1 + objs.append(pdb.stream(sha)) + if count == ni: + break + #END gather objects for pack-writing + elapsed = time() - st + print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + + st = time() + PackEntity.write_pack(objs, ostream.write) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + + def test_stream_reading(self): + raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # streaming only, meant for --with-profile runs From 03767cc17c6111c4c39cd4dde0517f5415515598 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 15:08:59 +0200 Subject: [PATCH 0140/1790] Initial setup for the testing framework; includes setup tools configuration and readme --- .gitignore | 6 +++++ Makefile | 37 +++++++++++++++++++++++++++++ README.rst | 50 +++++++++++++++++++++++++++++++++++++++ setup.py | 49 ++++++++++++++++++++++++++++++++++++++ smmap/__init__.py | 7 ++++++ smmap/mman.py | 3 +++ smmap/stream.py | 7 ++++++ smmap/test/__init__.py | 0 smmap/test/lib.py | 24 +++++++++++++++++++ smmap/test/test_mman.py | 7 ++++++ smmap/test/test_stream.py | 7 ++++++ 11 files changed, 197 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.rst create mode 100755 setup.py create mode 100644 smmap/__init__.py create mode 100644 smmap/mman.py create mode 100644 smmap/stream.py create mode 100644 smmap/test/__init__.py create mode 100644 smmap/test/lib.py create mode 100644 smmap/test/test_mman.py create mode 100644 smmap/test/test_stream.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..6cfb58df1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +build/ +.coverage +coverage +dist/ +MANIFEST diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ca3051dd4 --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +.PHONY: build sdist cover test clean-files clean-docs doc all + +all: + $(info Possible targets:) + $(info doc) + $(info clean-docs) + $(info clean-files) + $(info clean) + $(info test) + $(info coverage) + $(info build) + $(info sdist) + +doc: + cd docs && make html + +clean-docs: + cd docs && make clean + +clean-files: + git clean -fx + +clean: clean-files clean-docs + +test: + nosetests + +coverage: + nosetests --with-coverage --cover-package=smmap + +build: + ./setup.py build + +sdist: + ./setup.py sdist + + diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..08dc7111f --- /dev/null +++ b/README.rst @@ -0,0 +1,50 @@ +#################### +Sliding MMap (smmap) +#################### +A straight forward implementation of a slidinging memory map. +The idea is that every access to a file goes through a memory map manager, which will on demand map a region of a file and provide a string-like object for reading. + +When reading from it, you will have to check whether you are still within your window boundary, and possibly obtain a new window as required. + +The great benefit of this system is that you can use it to map files of any size even on 32 bit systems. Additionally it will be able to close unused windows right away to return system resources. If there are multiple clients for the same file and location, the same window will be reused as well. + +As there is a global management facility, you are also able to forcibly free all open handles which is handy on windows, which would otherwise prevent the deletion of the involved files. + +For convenience, a stream class is provided which hides the usage of the memory manager behind a simple stream interface. + +************ +LIMITATIONS +************ +The access is readonly by design. + +************ +REQUIREMENTS +************ +* Python 2.4 or higher + +******* +Install +******* +TODO + +****** +Source +****** +The source is available at git://github.com/Byron/smmap.git and can be cloned using:: + + git clone git://github.com/Byron/smmap.git + +************ +MAILING LIST +************ +http://groups.google.com/group/git-python + +************* +ISSUE TRACKER +************* +https://github.com/Byron/smmap/issues + +******* +LICENSE +******* +New BSD License diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..e2bb622fd --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import os +import codecs +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +import smmap + +if os.path.exists("README.rst"): + long_description = codecs.open('README.rst', "r", "utf-8").read() +else: + long_description = "See http://github.com/nvie/smmap/tree/master" + +setup( + name="smmap", + version=smmap.__version__, + description="A pure git implementation of a sliding window memory map manager", + author=smmap.__author__, + author_email=smmap.__contact__, + url=smmap.__homepage__, + platforms=["any"], + license="BSD", + packages=find_packages(), + zip_safe=True, + classifiers=[ + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", + #"Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Windows", + "Operating System :: OSX", + "Programming Language :: Python", + ], + long_description=long_description, +) diff --git a/smmap/__init__.py b/smmap/__init__.py new file mode 100644 index 000000000..82cff638c --- /dev/null +++ b/smmap/__init__.py @@ -0,0 +1,7 @@ +"""Intialize the smmap package""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/Byron/smmap" +version_info = (0, 8, 0) +__version__ = '.'.join(str(i) for i in version_info) diff --git a/smmap/mman.py b/smmap/mman.py new file mode 100644 index 000000000..dfa39db8c --- /dev/null +++ b/smmap/mman.py @@ -0,0 +1,3 @@ +"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" + +__all__ = [] diff --git a/smmap/stream.py b/smmap/stream.py new file mode 100644 index 000000000..bc5e568ea --- /dev/null +++ b/smmap/stream.py @@ -0,0 +1,7 @@ +"""Module with a simple stream implementation using the memory manager""" + +from mman import * + +__all__ = [] + + diff --git a/smmap/test/__init__.py b/smmap/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/smmap/test/lib.py b/smmap/test/lib.py new file mode 100644 index 000000000..450dd9dda --- /dev/null +++ b/smmap/test/lib.py @@ -0,0 +1,24 @@ +"""Provide base classes for the test system""" +from unittest import TestCase + +__all__ = ['TestBase'] + + +class TestBase(TestCase): + """Foundation used by all tests""" + + #{ Configuration + + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + #END overrides + + #{ Interface + + #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py new file mode 100644 index 000000000..43aef7771 --- /dev/null +++ b/smmap/test/test_mman.py @@ -0,0 +1,7 @@ +from lib import TestBase + +from smmap.mman import * + +class TestMMan(TestBase): + def test_basics(self): + assert False diff --git a/smmap/test/test_stream.py b/smmap/test/test_stream.py new file mode 100644 index 000000000..fae928d9a --- /dev/null +++ b/smmap/test/test_stream.py @@ -0,0 +1,7 @@ +from lib import TestBase + +from smmap.stream import * + +class TestStream(TestBase): + def test_basics(self): + assert False From b75a09b997e278e5351b60bc17b738cf62594fe0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 15:19:45 +0200 Subject: [PATCH 0141/1790] Added docs framework --- doc/.gitignore | 2 + doc/Makefile | 89 +++++++++++++++++++ doc/make.bat | 113 ++++++++++++++++++++++++ doc/source/changes.rst | 9 ++ doc/source/conf.py | 194 +++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 23 +++++ 6 files changed, 430 insertions(+) create mode 100644 doc/.gitignore create mode 100644 doc/Makefile create mode 100644 doc/make.bat create mode 100644 doc/source/changes.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..32060acdd --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +build +*.version_info diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..675ec2094 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/smmap.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/smmap.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 000000000..6900a2a46 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\smmap.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\smmap.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/doc/source/changes.rst b/doc/source/changes.rst new file mode 100644 index 000000000..ee17e0af0 --- /dev/null +++ b/doc/source/changes.rst @@ -0,0 +1,9 @@ +######### +Changelog +######### + +********** +v0.8.0 +********** + +- Initial Release diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..a0dac1166 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# smmap documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 8 15:14:25 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'smmap' +copyright = u'2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.8.0' +# The full version, including alpha/beta/rc tags. +release = '0.8.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'smmapdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'smmap.tex', u'smmap Documentation', + u'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..cb03044e0 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,23 @@ +.. smmap documentation master file, created by + sphinx-quickstart on Wed Jun 8 15:14:25 2011. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to smmap's documentation! +================================= +**smmap** is a pure python implementation of a sliding memory map to help unifying memory mapped access on 32 and 64 bit systems and to help managing resources more efficiently. + +Contents: + +.. toctree:: + :maxdepth: 2 + + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + From 2156e9ab02ea27289e6b26c11b024685b3816935 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 19:15:41 +0200 Subject: [PATCH 0142/1790] Implemented Window including test. Started Region implementation as well as test, but noticed that a critical feature, the mmap's offset, doesn't exist prior to python 2.6. Its total crap, so is python --- README.rst | 5 +- smmap/mman.py | 110 +++++++++++++++++++++++++++++++++++++++- smmap/test/lib.py | 43 +++++++++++++++- smmap/test/test_mman.py | 67 +++++++++++++++++++++++- 4 files changed, 219 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 08dc7111f..4c22eed2b 100644 --- a/README.rst +++ b/README.rst @@ -15,12 +15,13 @@ For convenience, a stream class is provided which hides the usage of the memory ************ LIMITATIONS ************ -The access is readonly by design. +* The access is readonly by design. +* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0. ************ REQUIREMENTS ************ -* Python 2.4 or higher +* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function. ******* Install diff --git a/smmap/mman.py b/smmap/mman.py index dfa39db8c..004987a8f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,3 +1,111 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -__all__ = [] +__all__ = ["MappedMemoryManager"] + +import os +import mmap + +from mmap import PAGESIZE + +#{ Utilities + +def align_to_page(num, round_up): + """Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / PAGESIZE) * PAGESIZE; + if round_up and (res != num): + res += PAGESIZE; + #END handle size + return res; + +#}END utilities + +class Window(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "Window(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region.ofs_begin(), region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + self.ofs = align_to_page(self.ofs, 0) + self.size = align_to_page(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class Region(object): + """Defines a mapped region of memory, aligned to pagesizes + :note: deallocates used region automatically on destruction""" + __slots__ = ( + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_nc', # number of clients using this region + '_uc' # total amount of usages + ) + + + def __init__(self, path, ofs, size): + """Initialize a region, allocate the memory map + :param path: path to the file to map + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._nc = 0 + self._uc = 0 + + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + try: + self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs) + finally: + os.close(fd) + #END close file handle + + +class MappedMemoryManager(object): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + :note: currently not thread-safe ! + :note: in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 450dd9dda..0605313d3 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -1,9 +1,50 @@ """Provide base classes for the test system""" from unittest import TestCase +import os +import tempfile -__all__ = ['TestBase'] +__all__ = ['TestBase', 'FileCreator'] +#{ Utilities + +class FileCreator(object): + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._path = tempfile.mktemp(prefix=prefix) + self._size = size + + fp = open(self._path, "wb") + fp.seek(size-1) + fp.write('1') + fp.close() + + assert os.path.getsize(self.path) == size + + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + #END exception handling + + + @property + def path(self): + return self._path + + @property + def size(self): + return self._size + +#} END utilities + class TestBase(TestCase): """Foundation used by all tests""" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 43aef7771..faee8c9cf 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,70 @@ -from lib import TestBase +from lib import TestBase, FileCreator from smmap.mman import * +from smmap.mman import Region +from smmap.mman import Window + +import sys +import mmap class TestMMan(TestBase): + + _window_test_size = 1000 * 1000 * 8 + 5195 + + def test_window(self): + wl = Window(0, 1) # left + wc = Window(1, 1) # center + wc2 = Window(10, 5) # another center + wr = Window(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 + + + def test_region(self): + fc = FileCreator(self._window_test_size, "window_test") + rfull = Region(fc.path, 0, fc.size) + + + + Window.from_region # todo + pass + def test_basics(self): - assert False + pass From 66a78db0127ed84e22cda5aed2009955765959e4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 20:26:51 +0200 Subject: [PATCH 0143/1790] Implemented MappedRegion type including test --- smmap/mman.py | 73 ++++++++++++++++++++++++++++++++++++++--- smmap/test/test_mman.py | 23 ++++++++++--- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 004987a8f..c5a388750 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -3,6 +3,7 @@ __all__ = ["MappedMemoryManager"] import os +import sys import mmap from mmap import PAGESIZE @@ -64,16 +65,22 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class Region(object): +class MappedRegion(object): """Defines a mapped region of memory, aligned to pagesizes :note: deallocates used region automatically on destruction""" - __slots__ = ( + __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_nc', # number of clients using this region - '_uc' # total amount of usages - ) + '_uc', # total amount of usages + '_ms' # actual size of the mapping + ] + _need_compat_layer = sys.version_info[1] < 6 + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + def __init__(self, path, ofs, size): """Initialize a region, allocate the memory map @@ -88,10 +95,66 @@ def __init__(self, path, ofs, size): fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: - self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs) + kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + corrected_size = size + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size, corrected_size), **kwargs) + + print len(self._mf) + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, size) + #END handle buffer wrapping finally: os.close(fd) #END close file handle + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return len(self._mf) + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self.size() + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + + def client_count(self): + """:return: number of clients currently using this region""" + return self._nc + + def adjust_client_count(self, ofs): + """Adjust the client count by the given positive or negative offset""" + self._nc += ofs + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def adjust_usage_count(self, ofs): + """Adjust the usage count by the given positive or negative offset""" + self._uc += ofs + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return len(self._mf) - self._b + + def ofs_end(self): + return len(self._mf) + #END handle compat layer class MappedMemoryManager(object): diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index faee8c9cf..482ef923c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import Region +from smmap.mman import MappedRegion from smmap.mman import Window import sys @@ -59,12 +59,27 @@ def test_window(self): def test_region(self): fc = FileCreator(self._window_test_size, "window_test") - rfull = Region(fc.path, 0, fc.size) + half_size = fc.size / 2 + rofs = 4000 + rfull = MappedRegion(fc.path, 0, fc.size) + rhalfofs = MappedRegion(fc.path, rofs, fc.size) + rhalfsize = MappedRegion(fc.path, 0, half_size) + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + + # window constructor + w = Window.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - Window.from_region # todo - pass def test_basics(self): pass From 0bf73877f860a86d9cf601154e9a9f76292e63c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 21:00:33 +0200 Subject: [PATCH 0144/1790] Fixed bug in test case as it didn't properly align its offset to a page --- smmap/mman.py | 20 +++++++++++++++----- smmap/test/test_mman.py | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index c5a388750..27904f5d0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -7,6 +7,7 @@ import mmap from mmap import PAGESIZE +from sys import getrefcount #{ Utilities @@ -71,7 +72,6 @@ class MappedRegion(object): __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) - '_nc', # number of clients using this region '_uc', # total amount of usages '_ms' # actual size of the mapping ] @@ -90,24 +90,24 @@ def __init__(self, path, ofs, size): allocated the the size automatically adjusted :raise Exception: if no memory can be allocated""" self._b = ofs - self._nc = 0 self._uc = 0 fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) corrected_size = size + sizeofs = ofs if self._need_compat_layer: del(kwargs['offset']) corrected_size += ofs + sizeofs = 0 # END handle python not supporting offset ! Arg # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size, corrected_size), **kwargs) + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) - print len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) #END handle buffer wrapping @@ -133,7 +133,8 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" - return self._nc + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 def adjust_client_count(self, ofs): """Adjust the client count by the given positive or negative offset""" @@ -157,6 +158,15 @@ def ofs_end(self): #END handle compat layer +class Cursor(object): + """Pointer into the mapped region of the memory manager, keeping the current window + alive until it is destroyed""" + + +class MappedRegionList(list): + """List of MappedRegion instances with specific functionality""" + + class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 482ef923c..00c9d90e8 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,8 +1,11 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MappedRegion +from smmap.mman import align_to_page from smmap.mman import Window +from smmap.mman import MappedRegion +from smmap.mman import MappedRegionList +from smmap.mman import Cursor import sys import mmap @@ -56,11 +59,10 @@ def test_window(self): wc.align() assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 - def test_region(self): fc = FileCreator(self._window_test_size, "window_test") half_size = fc.size / 2 - rofs = 4000 + rofs = align_to_page(4200, False) rfull = MappedRegion(fc.path, 0, fc.size) rhalfofs = MappedRegion(fc.path, rofs, fc.size) rhalfsize = MappedRegion(fc.path, 0, half_size) @@ -76,10 +78,20 @@ def test_region(self): assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + # window constructor w = Window.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + def test_region_list(self): + pass + + def test_cursor(self): + pass def test_basics(self): pass From 25e50356ab7c5392cece8132a8ee5b99c1789734 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 21:25:07 +0200 Subject: [PATCH 0145/1790] Added rather trivial implementation for the region list, including test --- smmap/mman.py | 30 ++++++++++++++++++++++++++---- smmap/test/test_mman.py | 7 ++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 27904f5d0..e0b4a5e64 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -158,15 +158,37 @@ def ofs_end(self): #END handle compat layer +class MappedRegionList(list): + """List of MappedRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path', # path which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MappedRegionList, cls).__new__(cls) + + def __init__(self, path): + self._path = path + self._file_size = None + + def path(self): + """:return: path to file whose regions we manage""" + return self._path + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + self._file_size = os.stat(self._path).st_size + #END update file size + return self._file_size + + class Cursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed""" - -class MappedRegionList(list): - """List of MappedRegion instances with specific functionality""" - class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 00c9d90e8..2027d9004 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -88,7 +88,12 @@ def test_region(self): assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): - pass + fc = FileCreator(100, "sample_file") + ml = MappedRegionList(fc.path) + + assert len(ml) == 0 + assert ml.path() == fc.path + assert ml.file_size() == fc.size def test_cursor(self): pass From cab0e3d6d995b7814e09157086d07c3ca2c901d4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 22:36:11 +0200 Subject: [PATCH 0146/1790] Moved all utility types into their own module --- smmap/mman.py | 204 ++++++---------------------------------- smmap/test/lib.py | 2 +- smmap/test/test_mman.py | 93 +----------------- smmap/test/test_util.py | 89 ++++++++++++++++++ smmap/util.py | 189 +++++++++++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 268 deletions(-) create mode 100644 smmap/test/test_util.py create mode 100644 smmap/util.py diff --git a/smmap/mman.py b/smmap/mman.py index e0b4a5e64..0ecc1aad1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,192 +1,46 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -__all__ = ["MappedMemoryManager"] +__all__ = ["MappedMemoryManager", "MemoryCursor"] -import os -import sys -import mmap - -from mmap import PAGESIZE -from sys import getrefcount - -#{ Utilities - -def align_to_page(num, round_up): - """Align the given integer number to the closest page offset, which usually is 4096 bytes. - :param round_up: if True, the next higher multiple of page size is used, otherwise - the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) - :return: num rounded to closest page""" - res = (num / PAGESIZE) * PAGESIZE; - if round_up and (res != num): - res += PAGESIZE; - #END handle size - return res; - -#}END utilities - -class Window(object): - """Utility type which is used to snap windows towards each other, and to adjust their size""" - __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes +from util import ( + MemoryWindow, + MappedRegion, + MappedRegionList, ) - def __init__(self, offset, size): - self.ofs = offset - self.size = size - - def __repr__(self): - return "Window(%i, %i)" % (self.ofs, self.size) - - @classmethod - def from_region(cls, region): - """:return: new window from a region""" - return cls(region.ofs_begin(), region.size()) - - def ofs_end(self): - return self.ofs + self.size - def align(self): - self.ofs = align_to_page(self.ofs, 0) - self.size = align_to_page(self.size, 1) - - def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, - but don't make yourself larger than max_size. - The resize will assure that the new window still contains the old window area""" - rofs = self.ofs - window.ofs_end() - nsize = rofs + self.size - rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs - self.size += rofs - - def extend_right_to(self, window, max_size): - """Adjust the size to make our window end where the right window begins, but don't - get larger than max_size""" - self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) - - -class MappedRegion(object): - """Defines a mapped region of memory, aligned to pagesizes - :note: deallocates used region automatically on destruction""" - __slots__ = [ - '_b' , # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_ms' # actual size of the mapping - ] - _need_compat_layer = sys.version_info[1] < 6 - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot - +class MemoryCursor(object): + """Pointer into the mapped region of the memory manager, keeping the current window + alive until it is destroyed""" + __slots__ = ( + '_manager', # the manger keeping all file regions + '_regions', # a regions list with regions for our file + '_region', # WEAK REF to our current region + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) - def __init__(self, path, ofs, size): - """Initialize a region, allocate the memory map - :param path: path to the file to map - :param ofs: **aligned** offset into the file to be mapped - :param size: if size is larger then the file on disk, the whole file will be - allocated the the size automatically adjusted - :raise Exception: if no memory can be allocated""" - self._b = ofs - self._uc = 0 - - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) - try: - kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) - corrected_size = size - sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will - # bark that the size is too large ... many extra file accesses because - # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, size) - #END handle buffer wrapping - finally: - os.close(fd) - #END close file handle - - def ofs_begin(self): - """:return: absolute byte offset to the first byte of the mapping""" - return self._b - - def size(self): - """:return: total size of the mapped region in bytes""" - return len(self._mf) + def __init__(self, manager = None, regions = None): + self._manager = manager + self._regions = regions + self._region = region + self._ofs = 0 + self._size = 0 - def ofs_end(self): - """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self.size() + def __del__(self): + self._destroy() - def includes_ofs(self, ofs): - """:return: True if the given offset can be read in our mapped region""" - return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + def _destroy(self): + """Destruction code to decrement counters""" - def client_count(self): - """:return: number of clients currently using this region""" - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" - def adjust_client_count(self, ofs): - """Adjust the client count by the given positive or negative offset""" - self._nc += ofs - - def usage_count(self): - """:return: amount of usages so far""" - return self._uc - - def adjust_usage_count(self, ofs): - """Adjust the usage count by the given positive or negative offset""" - self._uc += ofs - - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return len(self._mf) - self._b - - def ofs_end(self): - return len(self._mf) - #END handle compat layer + #{ Interface - -class MappedRegionList(list): - """List of MappedRegion instances associating a path with a list of regions.""" - __slots__ = ( - '_path', # path which is mapped by all our regions - '_file_size' # total size of the file we map - ) - - def __new__(cls, path): - return super(MappedRegionList, cls).__new__(cls) - def __init__(self, path): - self._path = path - self._file_size = None - - def path(self): - """:return: path to file whose regions we manage""" - return self._path - - def file_size(self): - """:return: size of file we manager""" - if self._file_size is None: - self._file_size = os.stat(self._path).st_size - #END update file size - return self._file_size + #} END interface - -class Cursor(object): - """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed""" class MappedMemoryManager(object): diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 0605313d3..6957dcab0 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -49,7 +49,7 @@ class TestBase(TestCase): """Foundation used by all tests""" #{ Configuration - + k_window_test_size = 1000 * 1000 * 8 + 5195 #} END configuration #{ Overrides diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 2027d9004..edc2ec2fe 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,102 +1,11 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import align_to_page -from smmap.mman import Window -from smmap.mman import MappedRegion -from smmap.mman import MappedRegionList -from smmap.mman import Cursor - -import sys -import mmap class TestMMan(TestBase): - _window_test_size = 1000 * 1000 * 8 + 5195 - - def test_window(self): - wl = Window(0, 1) # left - wc = Window(1, 1) # center - wc2 = Window(10, 5) # another center - wr = Window(8000, 50) # right - - assert wl.ofs_end() == 1 - assert wc.ofs_end() == 2 - assert wr.ofs_end() == 8050 - - # extension does nothing if already in place - maxsize = 100 - wc.extend_left_to(wl, maxsize) - assert wc.ofs == 1 and wc.size == 1 - wl.extend_right_to(wc, maxsize) - wl.extend_right_to(wc, maxsize) - assert wl.ofs == 0 and wl.size == 1 - - # an actual left extension - pofs_end = wc2.ofs_end() - wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - - # respects maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - - # without maxsize - wc.extend_right_to(wr, sys.maxint) - assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - - # extend left - wr.extend_left_to(wc2, maxsize) - wr.extend_left_to(wc2, maxsize) - assert wr.size == maxsize - - wr.extend_left_to(wc2, sys.maxint) - assert wr.ofs == wc2.ofs_end() - - wc.align() - assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 - - def test_region(self): - fc = FileCreator(self._window_test_size, "window_test") - half_size = fc.size / 2 - rofs = align_to_page(4200, False) - rfull = MappedRegion(fc.path, 0, fc.size) - rhalfofs = MappedRegion(fc.path, rofs, fc.size) - rhalfsize = MappedRegion(fc.path, 0, half_size) - - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always - - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - - # auto-refcount - assert rfull.client_count() == 1 - rfull2 = rfull - assert rfull.client_count() == 2 - - # window constructor - w = Window.from_region(rfull) - assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - - def test_region_list(self): - fc = FileCreator(100, "sample_file") - ml = MappedRegionList(fc.path) - - assert len(ml) == 0 - assert ml.path() == fc.path - assert ml.file_size() == fc.size - def test_cursor(self): - pass + man = MappedMemoryManager() def test_basics(self): pass diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py new file mode 100644 index 000000000..815391613 --- /dev/null +++ b/smmap/test/test_util.py @@ -0,0 +1,89 @@ +from lib import TestBase, FileCreator + +from smmap.util import * + +import sys + +class TestMMan(TestBase): + + def test_window(self): + wl = MemoryWindow(0, 1) # left + wc = MemoryWindow(1, 1) # center + wc2 = MemoryWindow(10, 5) # another center + wr = MemoryWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == PAGESIZE*2 + + def test_region(self): + fc = FileCreator(self.k_window_test_size, "window_test") + half_size = fc.size / 2 + rofs = align_to_page(4200, False) + rfull = MappedRegion(fc.path, 0, fc.size) + rhalfofs = MappedRegion(fc.path, rofs, fc.size) + rhalfsize = MappedRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + + # window constructor + w = MemoryWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + fc = FileCreator(100, "sample_file") + ml = MappedRegionList(fc.path) + + assert len(ml) == 0 + assert ml.path() == fc.path + assert ml.file_size() == fc.size + diff --git a/smmap/util.py b/smmap/util.py new file mode 100644 index 000000000..784227752 --- /dev/null +++ b/smmap/util.py @@ -0,0 +1,189 @@ +"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +import os +import sys +import mmap + +from mmap import PAGESIZE +from sys import getrefcount + +__all__ = ["align_to_page", "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] + +#{ Utilities + +def align_to_page(num, round_up): + """Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / PAGESIZE) * PAGESIZE; + if round_up and (res != num): + res += PAGESIZE; + #END handle size + return res; + +#}END utilities + + +#{ Utility Classes + +class MemoryWindow(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "MemoryWindow(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region.ofs_begin(), region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + self.ofs = align_to_page(self.ofs, 0) + self.size = align_to_page(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class MappedRegion(object): + """Defines a mapped region of memory, aligned to pagesizes + :note: deallocates used region automatically on destruction""" + __slots__ = [ + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_ms', # actual size of the mapping + '__weakref__' # allow weak references to a region + ] + _need_compat_layer = sys.version_info[1] < 6 + + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + + + def __init__(self, path, ofs, size): + """Initialize a region, allocate the memory map + :param path: path to the file to map + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._uc = 0 + + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + try: + kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + sizeofs = 0 + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) + + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, size) + #END handle buffer wrapping + finally: + os.close(fd) + #END close file handle + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return len(self._mf) + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self.size() + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + + def client_count(self): + """:return: number of clients currently using this region""" + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 + + def adjust_client_count(self, ofs): + """Adjust the client count by the given positive or negative offset""" + self._nc += ofs + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def adjust_usage_count(self, ofs): + """Adjust the usage count by the given positive or negative offset""" + self._uc += ofs + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return len(self._mf) - self._b + + def ofs_end(self): + return len(self._mf) + #END handle compat layer + + +class MappedRegionList(list): + """List of MappedRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path', # path which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MappedRegionList, cls).__new__(cls) + + def __init__(self, path): + self._path = path + self._file_size = None + + def path(self): + """:return: path to file whose regions we manage""" + return self._path + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + self._file_size = os.stat(self._path).st_size + #END update file size + return self._file_size + +#} END utilty classes From 9fe8f93439118c7ea6c4d5dece879b3849f0931e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 23:50:19 +0200 Subject: [PATCH 0147/1790] Implemented first basic functionality of cursor, which is only complete once some more of the memory manager is implemented. Next up is its major use_window method --- smmap/mman.py | 106 +++++++++++++++++++++++++++++++++++++--- smmap/test/test_mman.py | 15 +++++- smmap/test/test_util.py | 5 ++ smmap/util.py | 12 ++--- 4 files changed, 120 insertions(+), 18 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 0ecc1aad1..af414eada 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,29 +1,34 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" - -__all__ = ["MappedMemoryManager", "MemoryCursor"] - from util import ( MemoryWindow, MappedRegion, MappedRegionList, ) +from weakref import proxy + +__all__ = ["MappedMemoryManager"] +#{ Utilities + +#}END utilities class MemoryCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed""" + alive until it is destroyed. + + Cursors should not be created manually, but are instead returned by the MappedMemoryManager""" __slots__ = ( '_manager', # the manger keeping all file regions - '_regions', # a regions list with regions for our file - '_region', # WEAK REF to our current region + '_rlist', # a regions list with regions for our file + '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide ) def __init__(self, manager = None, regions = None): self._manager = manager - self._regions = regions - self._region = region + self._rlist = regions + self._region = None self._ofs = 0 self._size = 0 @@ -32,12 +37,97 @@ def __del__(self): def _destroy(self): """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._files.pop(self._rlist.path()) + #END remove regions list from manager + #END handle regions def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = rhs._rlist + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + if self._region is not None: + self._region.increment_usage_count(1) + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternativly, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset, size): + """Assure we point to a window which allows access to the given offset into the file + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed becaues we reached the end of the file + :note: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + + def unuse_region(self): + """Unuse the ucrrent region. Does nothing if we have no current region + :note: the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + self._region = None + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor""" + return self._region.ofs_begin() + self._ofs + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region_ref(self): + """:return: weak proxy to our mapped region. + :raise AssertionError: if we have no current region. This is only useful for debugging""" + if self._region is None: + raise AssertionError("region not set") + return proxy(self._region) + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + :note: always False if the cursor does not point to a valid region""" + if self._region is None: + return False + return (self.ofs_begin() <= ofs) and (ofs < self.ofs_end()) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path(self): + """:return: path of the underlying mapped file""" + return self._rlist.path() #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index edc2ec2fe..85a0a1bf5 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,11 +1,22 @@ from lib import TestBase, FileCreator +from copy import copy from smmap.mman import * +from smmap.mman import MemoryCursor class TestMMan(TestBase): def test_cursor(self): - man = MappedMemoryManager() + man = MappedMemoryManager() + c = MemoryCursor(man) + assert not c.is_valid() + assert not c.is_associated() - def test_basics(self): + # copy module + + # assign method + + + + def test_memory_manager(self): pass diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 815391613..2a0e0551e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -75,6 +75,11 @@ def test_region(self): rfull2 = rfull assert rfull.client_count() == 2 + # usage + assert rfull.usage_count() == 0 + rfull.increment_usage_count() + assert rfull.usage_count() == 1 + # window constructor w = MemoryWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() diff --git a/smmap/util.py b/smmap/util.py index 784227752..e4c5c82d6 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -75,8 +75,8 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages - '_ms', # actual size of the mapping - '__weakref__' # allow weak references to a region + '_ms' # actual size of the mapping + '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 @@ -139,17 +139,13 @@ def client_count(self): # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 - def adjust_client_count(self, ofs): - """Adjust the client count by the given positive or negative offset""" - self._nc += ofs - def usage_count(self): """:return: amount of usages so far""" return self._uc - def adjust_usage_count(self, ofs): + def increment_usage_count(self): """Adjust the usage count by the given positive or negative offset""" - self._uc += ofs + self._uc += 1 # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: From 226f42858faab545df0e0a9636c84791f0b3a080 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 01:13:52 +0200 Subject: [PATCH 0148/1790] First bit of MappedMemoryManager started. Much more to come - got lost in git++ and improved it in the meanwhile --- smmap/exc.py | 7 +++++++ smmap/mman.py | 22 +++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 smmap/exc.py diff --git a/smmap/exc.py b/smmap/exc.py new file mode 100644 index 000000000..a090d24d5 --- /dev/null +++ b/smmap/exc.py @@ -0,0 +1,7 @@ +"""Module with system exceptions""" + +class MemoryManagerError(Exception): + """Base class for all exceptions thrown by the memory manager""" + +class RegionCollectionError(MemoryManagerError): + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index af414eada..0d8c8dce4 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -5,6 +5,7 @@ MappedRegionList, ) +from exc import RegionCollectionError from weakref import proxy __all__ = ["MappedMemoryManager"] @@ -45,7 +46,7 @@ def _destroy(self): num_clients = self._rlist.client_count() - 2 if num_clients == 0 and len(self._rlist) == 0: # Free all resources associated with the mapped file - self._manager._files.pop(self._rlist.path()) + self._manager._fdict.pop(self._rlist.path()) #END remove regions list from manager #END handle regions @@ -82,6 +83,7 @@ def use_region(self, offset, size): This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region @@ -146,5 +148,19 @@ class MappedMemoryManager(object): a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" - __slots__ = tuple() - + __slots__ = [ + '_fdict', # mapping of path -> MappedRegionList + '_max_window_size', # maximum size of a window + '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_handles', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + def _collect_one_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :raise RegionCollectionError: + :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + From 04b9ec1e2c0bf657bf575a72a27acdbf2935ecf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 11:14:06 +0200 Subject: [PATCH 0149/1790] Fully implemented the manager - now the cursor can be implenented as well --- smmap/mman.py | 102 +++++++++++++++++++++++++++++++++++++++- smmap/test/test_mman.py | 22 ++++++++- smmap/test/test_util.py | 5 ++ smmap/util.py | 7 ++- 4 files changed, 131 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 0d8c8dce4..9c5432bd1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -3,6 +3,8 @@ MemoryWindow, MappedRegion, MappedRegionList, + is_64_bit, + PAGESIZE ) from exc import RegionCollectionError @@ -150,17 +152,113 @@ class MappedMemoryManager(object): __slots__ = [ '_fdict', # mapping of path -> MappedRegionList - '_max_window_size', # maximum size of a window + '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate - '_max_handles', # maximum amount of handles to keep open + '_max_handle_count', # maximum amount of handles to keep open '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles ] + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): + """initialize the manager with the given parameters. + :param window_size: if 0, a default window size will be chosen depending on + the operating system's architechture. It will internally be quantified to a multiple of the page size + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default iwll be set dependning on the system's architecture. + :param max_open_handles: if not ~0, lmit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, + the manager will free as many handles as posisble""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size == 0: + coeff = 32 + if is_64_bit(): + coeff = 1024 + #END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 512 + if is_64_bit(): + coeff = 8192 + #END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + #END handle max memory size + def _collect_one_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :raise RegionCollectionError: :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-1 == 0 and + (lru_region is None or region.usage_count() < lru_region.usage_count())): + lru_region = region + lru_list = regions + # END update lru_region + #END for each region + #END for each regions list + + if lru_region is None: + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + #END while there is more memory to free + + #{ Interface + def make_cursor(self, path): + """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" + regions = self._fdict.get(path) + if regions is None: + regions = MappedRegionList(path) + self._fdict[path] = regions + # END obtain region for path + return MemoryCursor(self, regions) + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + def page_size(self): + """:return: size of a single memory page in bytes""" + return PAGESIZE + + #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 85a0a1bf5..92c201c78 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,8 +1,13 @@ from lib import TestBase, FileCreator -from copy import copy from smmap.mman import * from smmap.mman import MemoryCursor +from smmap.util import PAGESIZE + +from smmap.exc import RegionCollectionError + +import sys +from copy import copy class TestMMan(TestBase): @@ -19,4 +24,17 @@ def test_cursor(self): def test_memory_manager(self): - pass + man = MappedMemoryManager() + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + assert man.window_size() > 0 + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + assert man.page_size() == PAGESIZE + + # collection doesn't raise in 'any' mode + man._collect_one_lru_region(0) + # doesn't raise if we are within the limit + man._collect_one_lru_region(10) + # raises outside of limit + self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 2a0e0551e..9866217d1 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -92,3 +92,8 @@ def test_region_list(self): assert ml.path() == fc.path assert ml.file_size() == fc.size + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_page(1, False) == 0 + assert align_to_page(1, True) == PAGESIZE + diff --git a/smmap/util.py b/smmap/util.py index e4c5c82d6..7f42834c4 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -6,7 +6,8 @@ from mmap import PAGESIZE from sys import getrefcount -__all__ = ["align_to_page", "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] +__all__ = [ "align_to_page", "is_64_bit", + "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] #{ Utilities @@ -20,6 +21,10 @@ def align_to_page(num, round_up): res += PAGESIZE; #END handle size return res; + +def is_64_bit(): + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxint > (1<<32) - 1 #}END utilities From e7b0e1c2c55c52b144671a1f3a8433901e42804a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 12:56:56 +0200 Subject: [PATCH 0150/1790] Implemented use_region. Let the testing begin. Especially the actual data handling will be interesting, which has to work exclusively through buffer objects. There are plenty of layers between the user and the data, which will always be copied when slicing it (as we have no memoryview). The latter one could be implemented on in case we use 2.7 at some point. Now, let the testing begin ! --- smmap/mman.py | 131 ++++++++++++++++++++++++++++++++++++++-- smmap/test/test_mman.py | 38 ++++++++++-- smmap/test/test_util.py | 2 + smmap/util.py | 16 ++++- 4 files changed, 174 insertions(+), 13 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9c5432bd1..1261fc16c 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -9,6 +9,7 @@ from exc import RegionCollectionError from weakref import proxy +import sys __all__ = ["MappedMemoryManager"] #{ Utilities @@ -77,7 +78,7 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size): + def use_region(self, offset, size, _is_recursive=False): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map @@ -85,15 +86,130 @@ def use_region(self, offset, size): This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + size = min(size, man.window_size()) # clamp size to window size + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + if need_region: + # abort on offsets beyond our mapped file's size - currently we are invalid + if offset > self.file_size(): + return self + # END handle offset too large + + existing_region = None + for region in self._rlist: + if region.includes_ofs(offset): + existing_region = region + break + #END handle existing region + #END for each existing region + if existing_region is None: + left = MemoryWindow(0, 0) + mid = MemoryWindow(offset, size) + right = MemoryWindow(self.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + man._collect_lru_region(man.window_size()) + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(self._rlist) + if len_regions == 1: + if self._rlist[0].ofs_begin() <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(self._rlist): + if region.ofs_begin() > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = MemoryWindow.from_region(self._rlist[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = MemoryWindow.from_region(self._rlist[insert_pos]) + # END adjust right window + left = MemoryWindow.from_region(self._rlist[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, man._window_size) + mid.extend_right_to(right, man._window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if man._handle_count >= man._max_handle_count: + raise Exception + #END assert own imposed max file handles + self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if _is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + man._collect_lru_region(0) + return self.use_region(offset, size, True) + #END handle exceptions + + man._handle_count += 1 + man._memory_size += self._region.size() + self._rlist.insert(insert_pos, self._region) + else: + self._region = existing_region + #END need region handling + #END handle acquire region + + self._region.increment_usage_count() + self._ofs = offset - self._region.ofs_begin() + self._size = min(size, self._region.ofs_end() - offset) + + return self + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region :note: the cursor unuses the region automatically upon destruction. It is recommended to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None - + + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested + :note: You can only obtain a buffer if this instance is_valid() !""" + return buffer(self._region.buffer(), self._ofs, self._size) + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None @@ -136,7 +252,6 @@ def path(self): #} END interface - class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. @@ -161,13 +276,13 @@ class MappedMemoryManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. :param window_size: if 0, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. - :param max_open_handles: if not ~0, lmit the amount of open file handles to the given number. + :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, the manager will free as many handles as posisble""" self._fdict = dict() @@ -193,7 +308,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size - def _collect_one_lru_region(self, size): + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region @@ -253,6 +368,10 @@ def mapped_memory_size(self): """:return: amount of bytes currently mapped in total""" return self._memory_size + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 92c201c78..97cd8f62b 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -12,16 +12,36 @@ class TestMMan(TestBase): def test_cursor(self): + fc = FileCreator(self.k_window_test_size, "cursor_test") + man = MappedMemoryManager() - c = MemoryCursor(man) - assert not c.is_valid() - assert not c.is_associated() + ci = MemoryCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated()# but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + # destruction is fine (even multiple times) + cv._destroy() + MemoryCursor(man)._destroy() def test_memory_manager(self): man = MappedMemoryManager() @@ -33,8 +53,14 @@ def test_memory_manager(self): assert man.page_size() == PAGESIZE # collection doesn't raise in 'any' mode - man._collect_one_lru_region(0) + man._collect_lru_region(0) # doesn't raise if we are within the limit - man._collect_one_lru_region(10) + man._collect_lru_region(10) # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint) + self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + c = man.make_cursor(fc.path) + assert c.use_region(10, 10).is_valid() diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 9866217d1..136d99102 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -88,6 +88,8 @@ def test_region_list(self): fc = FileCreator(100, "sample_file") ml = MappedRegionList(fc.path) + assert ml.client_count() == 1 + assert len(ml) == 0 assert ml.path() == fc.path assert ml.file_size() == fc.size diff --git a/smmap/util.py b/smmap/util.py index 7f42834c4..456562a5f 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -54,7 +54,10 @@ def ofs_end(self): return self.ofs + self.size def align(self): - self.ofs = align_to_page(self.ofs, 0) + """Assures the previous window area is contained in the new one""" + nofs = align_to_page(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs self.size = align_to_page(self.size, 1) def extend_left_to(self, window, max_size): @@ -123,6 +126,10 @@ def __init__(self, path, ofs, size): os.close(fd) #END close file handle + def buffer(self): + """:return: a sliceable buffer which can be used to access the mapped memory""" + return self._mf + def ofs_begin(self): """:return: absolute byte offset to the first byte of the mapping""" return self._b @@ -159,6 +166,9 @@ def size(self): def ofs_end(self): return len(self._mf) + + def buffer(self): + return self._mfb #END handle compat layer @@ -176,6 +186,10 @@ def __init__(self, path): self._path = path self._file_size = None + def client_count(self): + """:return: amount of clients which hold a reference to this instance""" + return getrefcount(self)-3 + def path(self): """:return: path to file whose regions we manage""" return self._path From 997a580c0a14cfa0bed11477cd64198199ef99b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 15:29:52 +0200 Subject: [PATCH 0151/1790] implemented plenty of operational testing. It shows that its not yet working properly. Intersting, it was so promising, and went just a little bit too smooth. There we go :) --- smmap/mman.py | 13 ++++-- smmap/test/test_mman.py | 100 ++++++++++++++++++++++++++++++++++++++-- smmap/util.py | 1 - 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 1261fc16c..8a9066cdc 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -8,7 +8,7 @@ ) from exc import RegionCollectionError -from weakref import proxy +from weakref import ref import sys __all__ = ["MappedMemoryManager"] @@ -100,7 +100,7 @@ def use_region(self, offset, size, _is_recursive=False): if need_region: # abort on offsets beyond our mapped file's size - currently we are invalid - if offset > self.file_size(): + if offset >= self.file_size(): return self # END handle offset too large @@ -222,16 +222,21 @@ def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor""" return self._region.ofs_begin() + self._ofs + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region.ofs_begin() + self._ofs + self._size + def size(self): """:return: amount of bytes we point to""" return self._size def region_ref(self): - """:return: weak proxy to our mapped region. + """:return: weak ref to our mapped region. :raise AssertionError: if we have no current region. This is only useful for debugging""" if self._region is None: raise AssertionError("region not set") - return proxy(self._region) + return ref(self._region) def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 97cd8f62b..f592fc129 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -2,10 +2,11 @@ from smmap.mman import * from smmap.mman import MemoryCursor -from smmap.util import PAGESIZE - +from smmap.util import PAGESIZE, align_to_page from smmap.exc import RegionCollectionError +from random import randint +from time import time import sys from copy import copy @@ -59,8 +60,101 @@ def test_memory_manager(self): # raises outside of limit self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) - # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") c = man.make_cursor(fc.path) assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + fc = FileCreator(self.k_window_test_size, "manager_operation_test") + data = open(fc.path, 'rb').read() + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15) + c = man.make_cursor(fc.path) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + size = man.window_size() / 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + assert c.size() == size + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # obtain second window, which spans the first part of the file - it is a still the same window + assert c.use_region(0, size-10).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == size-10 + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:size-10] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - size + overshoot + assert c.use_region(base_offset, size).is_valid() + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 15000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + while num_random_accesses: + num_random_accesses += 1 + base_offset = randint(0, fc.size - 1) + + # precondition + assert man.max_mapped_memory_size() >= man.mapped_memory_size() + assert man.max_file_handles() >= man.num_file_handles() + + assert c.use_region(base_offset, size).is_valid() + assert c.buffer()[:] == data[base_offset:base_offset+size] + memory_read += c.size() + + assert c.includes_ofs(base_offset) + assert c.includes_ofs(base_offset+c.size()-1) + assert not c.includes_ofs(base_offset+c.size()) + # END while we should do an access + elapsed = time() - st + mb = 1000 * 1000 + sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" + % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) + diff --git a/smmap/util.py b/smmap/util.py index 456562a5f..55e007554 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -83,7 +83,6 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages - '_ms' # actual size of the mapping '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 From c90cfef5597b403ae03edf021442f9f44139561f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 15:56:43 +0200 Subject: [PATCH 0152/1790] Fixed a few little issues, the random access test/perftest now work perfectly. Its not too slow either, 160MB/s compared to the 320MB/s that the c++ implementation gets in release mode. Its odd that for some reason, the Debug version of the c++ implementation is at 940MB/s, which is more like the performance i would have expected --- smmap/mman.py | 4 ++-- smmap/test/test_mman.py | 5 ++--- smmap/util.py | 11 +++++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 8a9066cdc..09fa8fcf9 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -244,7 +244,7 @@ def includes_ofs(self, ofs): :note: always False if the cursor does not point to a valid region""" if self._region is None: return False - return (self.ofs_begin() <= ofs) and (ofs < self.ofs_end()) + return self.ofs_begin() <= ofs < self.ofs_end() def file_size(self): """:return: size of the underlying file""" @@ -326,7 +326,7 @@ def _collect_lru_region(self, size): for regions in self._fdict.itervalues(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-1 == 0 and + if (region.client_count()-2 == 0 and (lru_region is None or region.usage_count() < lru_region.usage_count())): lru_region = region lru_list = regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index f592fc129..071615269 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -138,15 +138,14 @@ def test_memman_operation(self): st = time() while num_random_accesses: - num_random_accesses += 1 + num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) # precondition assert man.max_mapped_memory_size() >= man.mapped_memory_size() assert man.max_file_handles() >= man.num_file_handles() - assert c.use_region(base_offset, size).is_valid() - assert c.buffer()[:] == data[base_offset:base_offset+size] + assert c.buffer()[:] == data[base_offset:base_offset+c.size()] memory_read += c.size() assert c.includes_ofs(base_offset) diff --git a/smmap/util.py b/smmap/util.py index 55e007554..1aae4f534 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -116,7 +116,7 @@ def __init__(self, path, ofs, size): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) @@ -125,6 +125,11 @@ def __init__(self, path, ofs, size): os.close(fd) #END close file handle + def __repr__(self): + return "MappedRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface + def buffer(self): """:return: a sliceable buffer which can be used to access the mapped memory""" return self._mf @@ -143,7 +148,7 @@ def ofs_end(self): def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" - return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + return self.ofs_begin() <= ofs < self.ofs_end() def client_count(self): """:return: number of clients currently using this region""" @@ -170,6 +175,8 @@ def buffer(self): return self._mfb #END handle compat layer + #} END interface + class MappedRegionList(list): """List of MappedRegion instances associating a path with a list of regions.""" From 4f3d1ad43740fde43a321abcf59676f8ab9c693f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 16:52:15 +0200 Subject: [PATCH 0153/1790] Applied some optimizations for performance. Python is very easily overwhelmed with plenty of calls, causing too much overhead --- smmap/mman.py | 66 +++++++++++++++++++++++++---------------- smmap/test/test_mman.py | 28 ++++++++++------- smmap/util.py | 19 +++++++----- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 09fa8fcf9..c13ef6599 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -99,18 +99,30 @@ def use_region(self, offset, size, _is_recursive=False): # END check existing region if need_region: + window_size = man._window_size + # abort on offsets beyond our mapped file's size - currently we are invalid if offset >= self.file_size(): return self # END handle offset too large existing_region = None - for region in self._rlist: - if region.includes_ofs(offset): - existing_region = region - break - #END handle existing region - #END for each existing region + a = self._rlist + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + existing_region = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting if existing_region is None: left = MemoryWindow(0, 0) @@ -119,20 +131,23 @@ def use_region(self, offset, size, _is_recursive=False): # we want to honor the max memory size, and assure we have anough # memory available - man._collect_lru_region(man.window_size()) + # Save calls ! + if self._manager._memory_size + window_size > self._manager._max_memory_size: + man._collect_lru_region(window_size) + #END handle collection # we assume the list remains sorted by offset insert_pos = 0 - len_regions = len(self._rlist) + len_regions = len(a) if len_regions == 1: - if self._rlist[0].ofs_begin() <= offset: + if a[0]._b <= offset: insert_pos = 1 #END maintain sort else: # find insert position insert_pos = len_regions - for i, region in enumerate(self._rlist): - if region.ofs_begin() > offset: + for i, region in enumerate(a): + if region._b > offset: insert_pos = i break #END if insert position is correct @@ -143,17 +158,17 @@ def use_region(self, offset, size, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = MemoryWindow.from_region(self._rlist[insert_pos]) + right = MemoryWindow.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = MemoryWindow.from_region(self._rlist[insert_pos]) + right = MemoryWindow.from_region(a[insert_pos]) # END adjust right window - left = MemoryWindow.from_region(self._rlist[insert_pos - 1]) + left = MemoryWindow.from_region(a[insert_pos - 1]) #END adjust surrounding windows - mid.extend_left_to(left, man._window_size) - mid.extend_right_to(right, man._window_size) + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) mid.align() # it can happen that we align beyond the end of the file @@ -166,7 +181,7 @@ def use_region(self, offset, size, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size) + self._region = MappedRegion(a.path(), mid.ofs, mid.size) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -185,14 +200,14 @@ def use_region(self, offset, size, _is_recursive=False): man._handle_count += 1 man._memory_size += self._region.size() - self._rlist.insert(insert_pos, self._region) + a.insert(insert_pos, self._region) else: self._region = existing_region #END need region handling #END handle acquire region self._region.increment_usage_count() - self._ofs = offset - self._region.ofs_begin() + self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) return self @@ -220,12 +235,12 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor""" - return self._region.ofs_begin() + self._ofs + return self._region._b + self._ofs def ofs_end(self): """:return: offset to one past the last available byte""" # unroll method calls for performance ! - return self._region.ofs_begin() + self._ofs + self._size + return self._region._b + self._ofs + self._size def size(self): """:return: amount of bytes we point to""" @@ -241,10 +256,9 @@ def region_ref(self): def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - :note: always False if the cursor does not point to a valid region""" - if self._region is None: - return False - return self.ofs_begin() <= ofs < self.ofs_end() + :note: cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) def file_size(self): """:return: size of the underlying file""" @@ -327,7 +341,7 @@ def _collect_lru_region(self, size): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and - (lru_region is None or region.usage_count() < lru_region.usage_count())): + (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 071615269..6f3809979 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -127,9 +127,6 @@ def test_memman_operation(self): # remove mapped regions if we have to assert man.num_file_handles() == 2 - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - # iterate through the windows, verify data contents # this will trigger map collection after a while max_random_accesses = 15000 @@ -137,23 +134,32 @@ def test_memman_operation(self): memory_read = 0 st = time() + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles while num_random_accesses: num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) # precondition - assert man.max_mapped_memory_size() >= man.mapped_memory_size() - assert man.max_file_handles() >= man.num_file_handles() + assert max_mapped_memory_size >= mapped_memory_size() + assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, size).is_valid() - assert c.buffer()[:] == data[base_offset:base_offset+c.size()] - memory_read += c.size() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize - assert c.includes_ofs(base_offset) - assert c.includes_ofs(base_offset+c.size()-1) - assert not c.includes_ofs(base_offset+c.size()) + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = time() - st mb = 1000 * 1000 - sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" + sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() diff --git a/smmap/util.py b/smmap/util.py index 1aae4f534..6a2056b5d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,7 +3,7 @@ import sys import mmap -from mmap import PAGESIZE +from mmap import PAGESIZE, mmap, ACCESS_READ from sys import getrefcount __all__ = [ "align_to_page", "is_64_bit", @@ -48,7 +48,7 @@ def __repr__(self): @classmethod def from_region(cls, region): """:return: new window from a region""" - return cls(region.ofs_begin(), region.size()) + return cls(region._b, region._size) def ofs_end(self): return self.ofs + self.size @@ -83,6 +83,7 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages + '_size', # cached size of our memory map '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 @@ -100,11 +101,12 @@ def __init__(self, path, ofs, size): allocated the the size automatically adjusted :raise Exception: if no memory can be allocated""" self._b = ofs + self._size = 0 self._uc = 0 fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: - kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size sizeofs = ofs if self._need_compat_layer: @@ -116,7 +118,8 @@ def __init__(self, path, ofs, size): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + self._mf = mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + self._size = len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) @@ -126,7 +129,7 @@ def __init__(self, path, ofs, size): #END close file handle def __repr__(self): - return "MappedRegion<%i, %i>" % (self._b, self.size()) + return "MappedRegion<%i, %i>" % (self._b, self._size) #{ Interface @@ -140,15 +143,15 @@ def ofs_begin(self): def size(self): """:return: total size of the mapped region in bytes""" - return len(self._mf) + return self._size def ofs_end(self): """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self.size() + return self._b + self._size def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" - return self.ofs_begin() <= ofs < self.ofs_end() + return self._b <= ofs < self._b + self._size def client_count(self): """:return: number of clients currently using this region""" From 011343f6f43ea868c3efbd00a43757eb82196e1d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 17:46:24 +0200 Subject: [PATCH 0154/1790] Cursor can now take additional flags when opening the file handle for mapping. Renamed stream to buf, as the first item will be a buffer which uses the cursor underneath. We should add a stream for good measure though --- smmap/{stream.py => buf.py} | 0 smmap/mman.py | 10 +++++++--- smmap/test/{test_stream.py => test_buf.py} | 4 ++-- smmap/util.py | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) rename smmap/{stream.py => buf.py} (100%) rename smmap/test/{test_stream.py => test_buf.py} (54%) diff --git a/smmap/stream.py b/smmap/buf.py similarity index 100% rename from smmap/stream.py rename to smmap/buf.py diff --git a/smmap/mman.py b/smmap/mman.py index c13ef6599..57e2d81b5 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -78,10 +78,12 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, _is_recursive=False): + def use_region(self, offset, size, flags = 0, _is_recursive=False): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, @@ -106,6 +108,8 @@ def use_region(self, offset, size, _is_recursive=False): return self # END handle offset too large + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. existing_region = None a = self._rlist lo = 0 @@ -181,7 +185,7 @@ def use_region(self, offset, size, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(a.path(), mid.ofs, mid.size) + self._region = MappedRegion(a.path(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -195,7 +199,7 @@ def use_region(self, offset, size, _is_recursive=False): raise #END handle existing recursion man._collect_lru_region(0) - return self.use_region(offset, size, True) + return self.use_region(offset, size, flags, True) #END handle exceptions man._handle_count += 1 diff --git a/smmap/test/test_stream.py b/smmap/test/test_buf.py similarity index 54% rename from smmap/test/test_stream.py rename to smmap/test/test_buf.py index fae928d9a..231abcf5b 100644 --- a/smmap/test/test_stream.py +++ b/smmap/test/test_buf.py @@ -1,7 +1,7 @@ from lib import TestBase -from smmap.stream import * +from smmap.buf import * -class TestStream(TestBase): +class TestBuf(TestBase): def test_basics(self): assert False diff --git a/smmap/util.py b/smmap/util.py index 6c5282cb9..786622d9a 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -93,18 +93,19 @@ class MappedRegion(object): #END handle additional slot - def __init__(self, path, ofs, size): + def __init__(self, path, ofs, size, flags = 0): """Initialize a region, allocate the memory map :param path: path to the file to map :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. :raise Exception: if no memory can be allocated""" self._b = ofs self._size = 0 self._uc = 0 - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size From 8670443c3919f5e71e5864717d93eeb7921432ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 22:01:10 +0200 Subject: [PATCH 0155/1790] Implemented buffer interface and test to fully proove it. Its not yet fully properly implemented --- smmap/buf.py | 91 +++++++++++++++++++++++++++++++++++++++-- smmap/mman.py | 12 +++++- smmap/test/test_buf.py | 89 +++++++++++++++++++++++++++++++++++++++- smmap/test/test_mman.py | 14 +++++-- 4 files changed, 196 insertions(+), 10 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index bc5e568ea..f7db963ed 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,7 +1,92 @@ -"""Module with a simple stream implementation using the memory manager""" +"""Module with a simple buffer implementation using the memory manager""" +from mman import MemoryCursor -from mman import * +import sys -__all__ = [] +__all__ = ["MappedMemoryBuffer"] + +class MappedMemoryBuffer(object): + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by an underlying memory manager. + + A buffer, once initialized, stays put on providing access to eactly one path. + A custom interface allows you to change paths mid way, and to optimize + the resource usage. + + Please note that this type is only fully usable if you configure it with the + MappedMemoryManager to use. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at your given offset.""" + __slots__ = '_c' # our cursor + + #{ Configuration + # A subclass must provide an instance of a (usually global) MappedMemoryManager + manager = None + #}END configuration + + def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given path if given. + :param path: if not None, the path to the file you want to access + If None, you have call begin_access before using the buffer + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = MemoryCursor(self.manager) + assert self.manager is not None, "Require the cls.manager variable to be set in subclass" + if path and not self.begin_access(path, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset + + def __del__(self): + self.end_access() + + def __getitem__(self, i): + c = self._c + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + assert c.is_valid() # TODO: remove for performance + return c.buffer()[i] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + if c.ofs_begin() >= i and j < c.ofs_end(): + return c.buffer()[i:j] + raise NotImplementedError() + #{ Interface + + def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if path is empty or None the existing path will be used if possible. + :return: True if the buffer can be used""" + if path and (not self._c.is_associated() or self._c.path() != path): + self._c = self.manager.make_cursor(path) + #END get associated cursor + + # reuse existing cursors if possible + if self._c.is_associated(): + return self._c.use_region(offset, size, flags).is_valid() + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._c.unuse_region() + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/mman.py b/smmap/mman.py index 57e2d81b5..0fa9ef450 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -226,7 +226,9 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested - :note: You can only obtain a buffer if this instance is_valid() !""" + :note: You can only obtain a buffer if this instance is_valid() ! + :note: buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) def is_valid(self): @@ -336,6 +338,7 @@ def _collect_lru_region(self, size): :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :raise RegionCollectionError: + :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -365,6 +368,8 @@ def _collect_lru_region(self, size): self._handle_count -= 1 #END while there is more memory to free + return num_found + #{ Interface def make_cursor(self, path): """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" @@ -375,6 +380,11 @@ def make_cursor(self, path): # END obtain region for path return MemoryCursor(self, regions) + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + def num_file_handles(self): """:return: amount of file handles in use. Each mapped region uses one file handle""" return self._handle_count diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 231abcf5b..a26c8bb09 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,7 +1,92 @@ -from lib import TestBase +from lib import TestBase, FileCreator +from smmap.mman import MappedMemoryManager from smmap.buf import * +from random import randint +from time import time +import sys + +class TestBuffer(MappedMemoryBuffer): + #{ Configuration + manager = MappedMemoryManager() + #} END configuration + + class TestBuf(TestBase): + def test_basics(self): - assert False + self.failUnlessRaises(AssertionError, MappedMemoryBuffer) # needs subclass + fc = FileCreator(self.k_window_test_size, "buffer_test") + + # invalid paths fail upon construction + self.failUnlessRaises(OSError, TestBuffer, "somefile") # invalid file + self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size) # offset too large + + buf = TestBuffer() # can create uninitailized buffers + assert not buf.cursor().is_valid() and not buf.cursor().is_associated() + + # can call end access any time + buf.end_access() + buf.end_access() + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(fc.path, fc.size) == False + assert buf.begin_access(fc.path, offset) == True + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert buf.cursor().is_valid() + + # simple access + data = open(fc.path, 'rb').read() + assert data[offset] == buf[0] + assert data[offset:offset*2] == buf[0:offset] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + + man = TestBuffer.manager + assert man.num_file_handles() == 1 + + # PERFORMANCE + # blast away with rnadom access and a full mapping - we don't want to + # exagerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 5000 + num_accesses_left = max_num_accesses + + for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man): + TestBuffer.manager = manager + st = time() + buf = TestBuffer(fc.path) + assert manager.num_file_handles() == 1 + num_bytes = 0 + fsize = fc.size + while num_accesses_left: + num_accesses_left -= 1 + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + pos = randint(0, fsize) + assert buf[pos] == data[pos] + # END handle num accesses + buf.end_access() + assert manager.num_file_handles() == 1 + assert manager.collect() == 1 + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = 1000*1000 + sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 6f3809979..46a0f50fb 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -75,7 +75,8 @@ def test_memman_operation(self): assert len(data) == fc.size # small windows, a reasonable max memory. Not too many regions at once - man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15) + max_num_handles = 15 + man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) c = man.make_cursor(fc.path) # still empty (more about that is tested in test_memory_manager() @@ -129,7 +130,7 @@ def test_memman_operation(self): # iterate through the windows, verify data contents # this will trigger map collection after a while - max_random_accesses = 15000 + max_random_accesses = 5000 num_random_accesses = max_random_accesses memory_read = 0 st = time() @@ -160,6 +161,11 @@ def test_memman_operation(self): mb = 1000 * 1000 sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) - + # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 From b00bc5e4c0e8eb47d0bf59c0a2c1f5399f4c8f58 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:15:42 +0200 Subject: [PATCH 0156/1790] Finished implementation of buffer including a test which shows the performance should be usable in the real world. Its actually not too bad --- smmap/buf.py | 31 +++++++++++++++++---- smmap/mman.py | 5 +++- smmap/test/test_buf.py | 60 ++++++++++++++++++++++++----------------- smmap/test/test_mman.py | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index f7db963ed..68e7c33b9 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -44,19 +44,40 @@ def __del__(self): def __getitem__(self, i): c = self._c + assert c.is_valid() if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage - assert c.is_valid() # TODO: remove for performance - return c.buffer()[i] + return c.buffer()[i-c.ofs_begin()] def __getslice__(self, i, j): c = self._c # fast path, slice fully included - safes a concatenate operation and # should be the default - if c.ofs_begin() >= i and j < c.ofs_end(): - return c.buffer()[i:j] - raise NotImplementedError() + assert c.is_valid() + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i-b:j-b] + else: + l = j-i # total length + ofs = i + # keep tokens, and join afterwards. This is faster + # as it can preallocate the total amoint of space needed + # (and its verified the implementation does that) + # Question is whether the list allocation doesn't counteract this, + # but lets see ... + tokens = list() + tappend = tokens.append + + while l: + c.use_region(ofs, l) + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + tappend(d) + #END while there are bytes to read + return ''.join(tokens) + # END fast or slow path #{ Interface def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): diff --git a/smmap/mman.py b/smmap/mman.py index 0fa9ef450..588a63563 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -222,6 +222,8 @@ def unuse_region(self): to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! def buffer(self): """Return a buffer object which allows access to our memory region from our offset @@ -240,7 +242,8 @@ def is_associated(self): return self._rlist is not None def ofs_begin(self): - """:return: offset to the first byte pointed to by our cursor""" + """:return: offset to the first byte pointed to by our cursor + :note: only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index a26c8bb09..d7e6d6c96 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -61,32 +61,42 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 5000 - num_accesses_left = max_num_accesses - - for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man): + max_num_accesses = 1000 + for manager, man_id in ( (man, 'optimal'), + (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')): TestBuffer.manager = manager - st = time() buf = TestBuffer(fc.path) assert manager.num_file_handles() == 1 - num_bytes = 0 - fsize = fc.size - while num_accesses_left: - num_accesses_left -= 1 - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - pos = randint(0, fsize) - assert buf[pos] == data[pos] - # END handle num accesses - buf.end_access() - assert manager.num_file_handles() == 1 - assert manager.collect() == 1 - assert manager.num_file_handles() == 0 - elapsed = time() - st - mb = 1000*1000 - sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 46a0f50fb..b1c8f68eb 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -158,7 +158,7 @@ def test_memman_operation(self): assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = time() - st - mb = 1000 * 1000 + mb = float(1000 * 1000) sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) From 68fba826cdfd0de532ccf14c9ad94bf035a36e1b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:22:46 +0200 Subject: [PATCH 0157/1790] Optimized __getslice__ implementation a bit --- smmap/buf.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 68e7c33b9..d1be1b65a 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -61,22 +61,17 @@ def __getslice__(self, i, j): else: l = j-i # total length ofs = i - # keep tokens, and join afterwards. This is faster - # as it can preallocate the total amoint of space needed - # (and its verified the implementation does that) - # Question is whether the list allocation doesn't counteract this, - # but lets see ... - tokens = list() - tappend = tokens.append - + # Keeping tokens in a list could possible be faster, but the list + # overhead outweighs the benefits (tested) ! + md = str() while l: c.use_region(ofs, l) d = c.buffer()[:l] ofs += len(d) l -= len(d) - tappend(d) + md += d #END while there are bytes to read - return ''.join(tokens) + return md # END fast or slow path #{ Interface From c30e930bf06edc8f60ce3311d13f6a61de8ac0ce Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:57:55 +0200 Subject: [PATCH 0158/1790] Changed buffer implementation to use a cursor right away instead of taking a path and a manager. This makes it much more flexible, as it doesn't have to care about the manager anymore, making it easier to use and making clear that it is meant for use with a mapped memory manager implementation. --- smmap/buf.py | 44 ++++++++++++++++-------------------------- smmap/test/test_buf.py | 36 +++++++++++++++++----------------- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index d1be1b65a..b4f581316 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -7,35 +7,23 @@ class MappedMemoryBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into - memory of a mapped file. The mapping is controlled by an underlying memory manager. - - A buffer, once initialized, stays put on providing access to eactly one path. - A custom interface allows you to change paths mid way, and to optimize - the resource usage. - - Please note that this type is only fully usable if you configure it with the - MappedMemoryManager to use. + memory of a mapped file. The mapping is controlled by the provided cursor. The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at your given offset.""" + first byte at the offset you used during initialization or begin_access""" __slots__ = '_c' # our cursor - #{ Configuration - # A subclass must provide an instance of a (usually global) MappedMemoryManager - manager = None - #}END configuration - def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0): - """Initalize the instance to operate on the given path if given. - :param path: if not None, the path to the file you want to access - If None, you have call begin_access before using the buffer + def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size :param flags: Additional flags to be passed to os.open :raise ValueError: if the buffer could not achieve a valid state""" - self._c = MemoryCursor(self.manager) - assert self.manager is not None, "Require the cls.manager variable to be set in subclass" - if path and not self.begin_access(path, offset, size, flags): + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") # END handle offset @@ -75,19 +63,19 @@ def __getslice__(self, i, j): # END fast or slow path #{ Interface - def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): + def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. For more information no the parameters, see the __init__ method - :param path: if path is empty or None the existing path will be used if possible. + :param path: if cursor is None the existing one will be used. :return: True if the buffer can be used""" - if path and (not self._c.is_associated() or self._c.path() != path): - self._c = self.manager.make_cursor(path) - #END get associated cursor + if cursor: + self._c = cursor + #END update our cursor # reuse existing cursors if possible - if self._c.is_associated(): + if self._c is not None and self._c.is_associated(): return self._c.use_region(offset, size, flags).is_valid() return False @@ -97,7 +85,9 @@ def end_access(self): resources to be freed. Once you called end_access, you must call begin access before reusing this instance!""" - self._c.unuse_region() + if self._c is not None: + self._c.unuse_region() + #END unuse region def cursor(self): """:return: the currently set cursor which provides access to the data""" diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d7e6d6c96..6192897bd 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -7,24 +7,24 @@ from time import time import sys -class TestBuffer(MappedMemoryBuffer): - #{ Configuration - manager = MappedMemoryManager() - #} END configuration - + +man_optimal = MappedMemoryManager() +man_worst_case = MappedMemoryManager( window_size=TestBase.k_window_test_size/100, + max_memory_size=TestBase.k_window_test_size/3, + max_open_handles=15) class TestBuf(TestBase): def test_basics(self): - self.failUnlessRaises(AssertionError, MappedMemoryBuffer) # needs subclass fc = FileCreator(self.k_window_test_size, "buffer_test") # invalid paths fail upon construction - self.failUnlessRaises(OSError, TestBuffer, "somefile") # invalid file - self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size) # offset too large + c = man_optimal.make_cursor(fc.path) + self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size) # offset too large - buf = TestBuffer() # can create uninitailized buffers - assert not buf.cursor().is_valid() and not buf.cursor().is_associated() + buf = MappedMemoryBuffer() # can create uninitailized buffers + assert buf.cursor() is None # can call end access any time buf.end_access() @@ -32,8 +32,9 @@ def test_basics(self): # begin access can revive it, if the offset is suitable offset = 100 - assert buf.begin_access(fc.path, fc.size) == False - assert buf.begin_access(fc.path, offset) == True + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True @@ -52,9 +53,9 @@ def test_basics(self): # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically + del(c) - man = TestBuffer.manager - assert man.num_file_handles() == 1 + assert man_optimal.num_file_handles() == 1 # PERFORMANCE # blast away with rnadom access and a full mapping - we don't want to @@ -62,10 +63,9 @@ def test_basics(self): # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 1000 - for manager, man_id in ( (man, 'optimal'), - (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')): - TestBuffer.manager = manager - buf = TestBuffer(fc.path) + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case')): + buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses From aafc980e9cbb1b2ed08d374300e9916538136cc7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 00:02:00 +0200 Subject: [PATCH 0159/1790] Added indirection level to internally used types to allow others to exchange them with their own implementations. --- smmap/mman.py | 25 +++++++++++++++++-------- smmap/test/test_buf.py | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 588a63563..d8626d065 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -29,6 +29,11 @@ class MemoryCursor(object): '_size' # maximum size we should provide ) + #{ Configuration + MemoryWindowCls = MemoryWindow + MappedRegionCls = MappedRegion + #} END configuration + def __init__(self, manager = None, regions = None): self._manager = manager self._rlist = regions @@ -129,9 +134,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): #END while bisecting if existing_region is None: - left = MemoryWindow(0, 0) - mid = MemoryWindow(offset, size) - right = MemoryWindow(self.file_size(), 0) + left = self.MemoryWindowCls(0, 0) + mid = self.MemoryWindowCls(offset, size) + right = self.MemoryWindowCls(self.file_size(), 0) # we want to honor the max memory size, and assure we have anough # memory available @@ -162,13 +167,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = MemoryWindow.from_region(a[insert_pos]) + right = self.MemoryWindowCls.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = MemoryWindow.from_region(a[insert_pos]) + right = self.MemoryWindowCls.from_region(a[insert_pos]) # END adjust right window - left = MemoryWindow.from_region(a[insert_pos - 1]) + left = self.MemoryWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows mid.extend_left_to(left, window_size) @@ -185,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(a.path(), mid.ofs, mid.size, flags) + self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -302,6 +307,10 @@ class MappedMemoryManager(object): '_handle_count', # amount of currently allocated file handles ] + #{ Configuration + MappedRegionListCls = MappedRegionList + #} END configuration + _MB_in_bytes = 1024 * 1024 def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): @@ -378,7 +387,7 @@ def make_cursor(self, path): """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" regions = self._fdict.get(path) if regions is None: - regions = MappedRegionList(path) + regions = self.MappedRegionListCls(path) self._fdict[path] = regions # END obtain region for path return MemoryCursor(self, regions) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 6192897bd..ae1a174d5 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -62,7 +62,7 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 1000 + max_num_accesses = 400 for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case')): buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) From 8d64e74ed80f2818acad652a69615708f8f61104 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 00:38:48 +0200 Subject: [PATCH 0160/1790] Added very special purpose method to free memory maps. The whole reason for this is to make the windows test work after all \! --- smmap/mman.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/smmap/mman.py b/smmap/mman.py index d8626d065..44d985e28 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -426,3 +426,32 @@ def page_size(self): return PAGESIZE #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + :note: does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface From 4ad22313d8764bcc3645ba8d8ea3ff6f50bba7d1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 09:42:55 +0200 Subject: [PATCH 0161/1790] System can now deal with file descriptors as input, but it still requires some more testing. Also fds need to remain open to be usable for new mapped regions --- smmap/mman.py | 41 ++++++++++++++++++++++++++++++++--------- smmap/test/test_util.py | 18 ++++++++++++------ smmap/util.py | 29 +++++++++++++++++++---------- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 44d985e28..79f2de896 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -54,7 +54,7 @@ def _destroy(self): num_clients = self._rlist.client_count() - 2 if num_clients == 0 and len(self._rlist) == 0: # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path()) + self._manager._fdict.pop(self._rlist.path_or_fd()) #END remove regions list from manager #END handle regions @@ -190,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags) + self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -278,9 +278,26 @@ def file_size(self): """:return: size of the underlying file""" return self._rlist.file_size() + def path_or_fd(self): + """:return: path or file decriptor of the underlying mapped file""" + return self._rlist.path_or_fd() + def path(self): - """:return: path of the underlying mapped file""" - return self._rlist.path() + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + :note: it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), basestring): + return ValueError("File descriptor queried although mapping was generated from path") + #END handle type + return self._rlist.path_or_fd() #} END interface @@ -383,12 +400,18 @@ def _collect_lru_region(self, size): return num_found #{ Interface - def make_cursor(self, path): - """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" - regions = self._fdict.get(path) + def make_cursor(self, path_or_fd): + """:return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + :note: if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open.""" + regions = self._fdict.get(path_or_fd) if regions is None: - regions = self.MappedRegionListCls(path) - self._fdict[path] = regions + regions = self.MappedRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions # END obtain region for path return MemoryCursor(self, regions) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 136d99102..a5478cd7c 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -2,6 +2,7 @@ from smmap.util import * +import os import sys class TestMMan(TestBase): @@ -86,13 +87,18 @@ def test_region(self): def test_region_list(self): fc = FileCreator(100, "sample_file") - ml = MappedRegionList(fc.path) - assert ml.client_count() == 1 - - assert len(ml) == 0 - assert ml.path() == fc.path - assert ml.file_size() == fc.size + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + ml = MappedRegionList(item) + + assert ml.client_count() == 1 + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + #END handle input + os.close(fd) def test_util(self): assert isinstance(is_64_bit(), bool) # just call it diff --git a/smmap/util.py b/smmap/util.py index 786622d9a..f3bb58b33 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -93,9 +93,9 @@ class MappedRegion(object): #END handle additional slot - def __init__(self, path, ofs, size, flags = 0): + def __init__(self, path_or_fd, ofs, size, flags = 0): """Initialize a region, allocate the memory map - :param path: path to the file to map + :param path_or_fd: path to the file to map, or the opened file descriptor :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted @@ -105,7 +105,12 @@ def __init__(self, path, ofs, size, flags = 0): self._size = 0 self._uc = 0 - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + #END handle fd + try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size @@ -189,29 +194,33 @@ def includes_ofs(self, ofs): class MappedRegionList(list): """List of MappedRegion instances associating a path with a list of regions.""" __slots__ = ( - '_path', # path which is mapped by all our regions + '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): return super(MappedRegionList, cls).__new__(cls) - def __init__(self, path): - self._path = path + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd self._file_size = None def client_count(self): """:return: amount of clients which hold a reference to this instance""" return getrefcount(self)-3 - def path(self): - """:return: path to file whose regions we manage""" - return self._path + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd def file_size(self): """:return: size of file we manager""" if self._file_size is None: - self._file_size = os.stat(self._path).st_size + if isinstance(self._path_or_fd, basestring): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + #END handle path type #END update file size return self._file_size From a8a5e10835d71bb99993744777a06fc5573c892c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 09:53:10 +0200 Subject: [PATCH 0162/1790] Added tests for the fd case. It shows that this is measurably faster than the string path version because there is less system overhead. Good to know actally --- smmap/mman.py | 4 +- smmap/test/test_buf.py | 80 +++++++++------- smmap/test/test_mman.py | 207 +++++++++++++++++++++------------------- smmap/util.py | 4 +- 4 files changed, 157 insertions(+), 138 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 79f2de896..b15b9af7c 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -407,7 +407,9 @@ def make_cursor(self, path_or_fd): but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open.""" + are preferred unless you plan to keep the file descriptor open. + :note: Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: regions = self.MappedRegionListCls(path_or_fd) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index ae1a174d5..efc1da60c 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -6,6 +6,7 @@ from random import randint from time import time import sys +import os man_optimal = MappedMemoryManager() @@ -63,40 +64,45 @@ def test_basics(self): # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 400 - for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case')): - buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - #END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = time() - st - mb = float(1000*1000) - mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) - # END handle access mode - # END for each manager + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case')): + buf = MappedMemoryBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index b1c8f68eb..57d78d504 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -7,6 +7,7 @@ from random import randint from time import time +import os import sys from copy import copy @@ -62,110 +63,118 @@ def test_memory_manager(self): # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") - c = man.make_cursor(fc.path) - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + #END for each input + os.close(fd) def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") data = open(fc.path, 'rb').read() - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - max_num_handles = 15 - man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) - c = man.make_cursor(fc.path) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - assert c.size() == size - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) - - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # obtain second window, which spans the first part of the file - it is a still the same window - assert c.use_region(0, size-10).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == size-10 - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:size-10] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - size + overshoot - assert c.use_region(base_offset, size).is_valid() - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + max_num_handles = 15 + man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) + c = man.make_cursor(item) - # precondition - assert max_mapped_memory_size >= mapped_memory_size() - assert max_file_handles >= num_file_handles() + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + size = man.window_size() / 2 assert c.use_region(base_offset, size).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = time() - st - mb = float(1000 * 1000) - sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" - % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + assert c.size() == size + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # obtain second window, which spans the first part of the file - it is a still the same window + assert c.use_region(0, size-10).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == size-10 + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:size-10] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - size + overshoot + assert c.use_region(base_offset, size).is_valid() + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + assert max_mapped_memory_size >= mapped_memory_size() + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, size).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = time() - st + mb = float(1000 * 1000) + sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + os.close(fd) diff --git a/smmap/util.py b/smmap/util.py index f3bb58b33..2ba7a1d29 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -131,7 +131,9 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._mfb = buffer(self._mf, ofs, size) #END handle buffer wrapping finally: - os.close(fd) + if isinstance(path_or_fd, basestring): + os.close(fd) + #END only close it if we opened it #END close file handle def __repr__(self): From 9f16040fb4cde025875e28b01d70cfba11c1d773 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 10:42:32 +0200 Subject: [PATCH 0163/1790] Fixed some mapping issues on windows. Fixed some tests to deal with the very different granularity --- smmap/mman.py | 5 ----- smmap/test/test_buf.py | 2 +- smmap/test/test_mman.py | 7 +++---- smmap/test/test_util.py | 17 ++++++++++++----- smmap/util.py | 16 ++++++++-------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index b15b9af7c..8e50a1445 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MappedRegion, MappedRegionList, is_64_bit, - PAGESIZE ) from exc import RegionCollectionError @@ -446,10 +445,6 @@ def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size - def page_size(self): - """:return: size of a single memory page in bytes""" - return PAGESIZE - #} END interface #{ Special Purpose Interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index efc1da60c..c772e4999 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -97,7 +97,7 @@ def test_basics(self): assert manager.num_file_handles() assert manager.collect() assert manager.num_file_handles() == 0 - elapsed = time() - st + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 57d78d504..e220f8bb2 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -2,7 +2,7 @@ from smmap.mman import * from smmap.mman import MemoryCursor -from smmap.util import PAGESIZE, align_to_page +from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError from random import randint @@ -52,7 +52,6 @@ def test_memory_manager(self): assert man.window_size() > 0 assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 - assert man.page_size() == PAGESIZE # collection doesn't raise in 'any' mode man._collect_lru_region(0) @@ -102,7 +101,7 @@ def test_memman_operation(self): assert c.size() == size assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) assert c.buffer()[:] == data[base_offset:base_offset+size] @@ -164,7 +163,7 @@ def test_memman_operation(self): assert includes_ofs(base_offset+csize-1) assert not includes_ofs(base_offset+csize) # END while we should do an access - elapsed = time() - st + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index a5478cd7c..7caa427ba 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,6 +1,7 @@ from lib import TestBase, FileCreator from smmap.util import * +from mmap import ALLOCATIONGRANULARITY import os import sys @@ -50,12 +51,12 @@ def test_window(self): assert wr.ofs == wc2.ofs_end() wc.align() - assert wc.ofs == 0 and wc.size == PAGESIZE*2 + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size / 2 - rofs = align_to_page(4200, False) + rofs = align_to_mmap(4200, False) rfull = MappedRegion(fc.path, 0, fc.size) rhalfofs = MappedRegion(fc.path, rofs, fc.size) rhalfsize = MappedRegion(fc.path, 0, half_size) @@ -69,7 +70,13 @@ def test_region(self): assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + # with the values we have, this test only works on windows where an alignment + # size of 4096 is assumed. + if sys.platform == 'win32': + assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0) + else: + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + #END handle platforms # auto-refcount assert rfull.client_count() == 1 @@ -102,6 +109,6 @@ def test_region_list(self): def test_util(self): assert isinstance(is_64_bit(), bool) # just call it - assert align_to_page(1, False) == 0 - assert align_to_page(1, True) == PAGESIZE + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY diff --git a/smmap/util.py b/smmap/util.py index 2ba7a1d29..e1f786538 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,22 +3,22 @@ import sys import mmap -from mmap import PAGESIZE, mmap, ACCESS_READ +from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ from sys import getrefcount -__all__ = [ "align_to_page", "is_64_bit", - "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] +__all__ = [ "align_to_mmap", "is_64_bit", + "MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities -def align_to_page(num, round_up): +def align_to_mmap(num, round_up): """Align the given integer number to the closest page offset, which usually is 4096 bytes. :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num / PAGESIZE) * PAGESIZE; + res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; if round_up and (res != num): - res += PAGESIZE; + res += ALLOCATIONGRANULARITY #END handle size return res; @@ -55,10 +55,10 @@ def ofs_end(self): def align(self): """Assures the previous window area is contained in the new one""" - nofs = align_to_page(self.ofs, 0) + nofs = align_to_mmap(self.ofs, 0) self.size += self.ofs - nofs # keep size constant self.ofs = nofs - self.size = align_to_page(self.size, 1) + self.size = align_to_mmap(self.size, 1) def extend_left_to(self, window, max_size): """Adjust the offset to start where the given window on our left ends if possible, From 4466476cf576cf5936a11524e67345a80e2ec5a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 10:48:28 +0200 Subject: [PATCH 0164/1790] Fixed missing ALLOCATIONGRANULARIY in python <2.6. Python is as unportable as ever across versions with simple functionality --- smmap/test/test_util.py | 1 - smmap/util.py | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 7caa427ba..4043cd83f 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,7 +1,6 @@ from lib import TestBase, FileCreator from smmap.util import * -from mmap import ALLOCATIONGRANULARITY import os import sys diff --git a/smmap/util.py b/smmap/util.py index e1f786538..667777861 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,7 +3,15 @@ import sys import mmap -from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ +from mmap import mmap, ACCESS_READ +try: + from mmap import ALLOCATIONGRANULARITY +except ImportError: + # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly + # useful for aligning the offset. The offset argument doesn't exist there though + from mmap import PAGESIZE as ALLOCATIONGRANULARITY +#END handle pythons missing quality assurance + from sys import getrefcount __all__ = [ "align_to_mmap", "is_64_bit", From 064aa81076b8b2e08209aa6525ca22065703b41d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 12:35:32 +0200 Subject: [PATCH 0165/1790] Implemented __len__ method in buffer, including small test. This has its caveats, but should be fine for responsible clients --- smmap/buf.py | 26 ++++++++++++++++++++++++-- smmap/test/test_buf.py | 5 ++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index b4f581316..4fc78ea6f 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -11,7 +11,10 @@ class MappedMemoryBuffer(object): The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access""" - __slots__ = '_c' # our cursor + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): @@ -20,6 +23,10 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! :param flags: Additional flags to be passed to os.open :raise ValueError: if the buffer could not achieve a valid state""" self._c = cursor @@ -30,6 +37,9 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): def __del__(self): self.end_access() + def __len__(self): + return self._size + def __getitem__(self, i): c = self._c assert c.is_valid() @@ -76,7 +86,18 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): - return self._c.use_region(offset, size, flags).is_valid() + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + #END handle size + self._size = size + #END set size + return res return False def end_access(self): @@ -85,6 +106,7 @@ def end_access(self): resources to be freed. Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 if self._c is not None: self._c.unuse_region() #END unuse region diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index c772e4999..48aeabb44 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -30,15 +30,18 @@ def test_basics(self): # can call end access any time buf.end_access() buf.end_access() + assert len(buf) == 0 # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True + assert len(buf) == fc.size assert buf.cursor().is_valid() # simple access @@ -63,7 +66,7 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 400 + max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), From d7b486df99139ff867db01f6427408a12ff213b3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:26:46 +0200 Subject: [PATCH 0166/1790] Added smmap as submodule, assured the sys path makes it available --- .gitmodules | 4 ++++ gitdb/__init__.py | 16 +++++++++------- gitdb/ext/smmap | 1 + 3 files changed, 14 insertions(+), 7 deletions(-) create mode 160000 gitdb/ext/smmap diff --git a/.gitmodules b/.gitmodules index 3db4c676d..1be8ccac9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = gitdb/ext/async url = git://github.com/gitpython-developers/async.git branch = master +[submodule "smmap"] + path = gitdb/ext/smmap + url = git://github.com/Byron/smmap.git + branch = master diff --git a/gitdb/__init__.py b/gitdb/__init__.py index a551f37dc..775c969cf 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -10,13 +10,15 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async')) - - try: - import async - except ImportError: - raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH") - #END verify import + for module in ('async', 'smmap'): + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + + try: + __import__(module) + except ImportError: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + #END verify import + #END handel imports #} END initialization diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap new file mode 160000 index 000000000..4466476cf --- /dev/null +++ b/gitdb/ext/smmap @@ -0,0 +1 @@ +Subproject commit 4466476cf576cf5936a11524e67345a80e2ec5a9 From d09158f9029c564c97cc33f173efd376eca873d1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:36:53 +0200 Subject: [PATCH 0167/1790] Made all types available in root package --- smmap/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/smmap/__init__.py b/smmap/__init__.py index 82cff638c..769858fa5 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -5,3 +5,7 @@ __homepage__ = "https://github.com/Byron/smmap" version_info = (0, 8, 0) __version__ = '.'.join(str(i) for i in version_info) + +# make everything available in root package for convenience +from mman import * +from buf import * From 9dc4a8dd154d15a243cd2d54f7d1631a913106f0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:42:05 +0200 Subject: [PATCH 0168/1790] Changed names to be more descriptive, hopefully. This opens op the option to implement such a manager differently, without the sliding window mechanics, which would be quite simple and not much better than a map of mmaps in the end --- smmap/buf.py | 6 +++--- smmap/mman.py | 40 ++++++++++++++++++++-------------------- smmap/test/test_buf.py | 14 +++++++------- smmap/test/test_mman.py | 12 ++++++------ smmap/test/test_util.py | 18 +++++++++--------- smmap/util.py | 16 ++++++++-------- 6 files changed, 53 insertions(+), 53 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 4fc78ea6f..94650a50c 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,11 +1,11 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import MemoryCursor +from mman import SlidingCursor import sys -__all__ = ["MappedMemoryBuffer"] +__all__ = ["SlidingWindowMapBuffer"] -class MappedMemoryBuffer(object): +class SlidingWindowMapBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. diff --git a/smmap/mman.py b/smmap/mman.py index 8e50a1445..312612298 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,8 +1,8 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" from util import ( - MemoryWindow, - MappedRegion, - MappedRegionList, + MapWindow, + MapRegion, + MapRegionList, is_64_bit, ) @@ -10,16 +10,16 @@ from weakref import ref import sys -__all__ = ["MappedMemoryManager"] +__all__ = ["SlidingWindowMapManager"] #{ Utilities #}END utilities -class MemoryCursor(object): +class SlidingCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed. - Cursors should not be created manually, but are instead returned by the MappedMemoryManager""" + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -29,8 +29,8 @@ class MemoryCursor(object): ) #{ Configuration - MemoryWindowCls = MemoryWindow - MappedRegionCls = MappedRegion + MapWindowCls = MapWindow + MapRegionCls = MapRegion #} END configuration def __init__(self, manager = None, regions = None): @@ -133,9 +133,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): #END while bisecting if existing_region is None: - left = self.MemoryWindowCls(0, 0) - mid = self.MemoryWindowCls(offset, size) - right = self.MemoryWindowCls(self.file_size(), 0) + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(self.file_size(), 0) # we want to honor the max memory size, and assure we have anough # memory available @@ -166,13 +166,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = self.MemoryWindowCls.from_region(a[insert_pos]) + right = self.MapWindowCls.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = self.MemoryWindowCls.from_region(a[insert_pos]) + right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window - left = self.MemoryWindowCls.from_region(a[insert_pos - 1]) + left = self.MapWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows mid.extend_left_to(left, window_size) @@ -189,7 +189,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -301,7 +301,7 @@ def fd(self): #} END interface -class MappedMemoryManager(object): +class SlidingWindowMapManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. Once a certain memory limit is reached globally, or if there cannot be more open file handles @@ -315,7 +315,7 @@ class MappedMemoryManager(object): space is full.""" __slots__ = [ - '_fdict', # mapping of path -> MappedRegionList + '_fdict', # mapping of path -> MapRegionList '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate '_max_handle_count', # maximum amount of handles to keep open @@ -324,7 +324,7 @@ class MappedMemoryManager(object): ] #{ Configuration - MappedRegionListCls = MappedRegionList + MapRegionListCls = MapRegionList #} END configuration _MB_in_bytes = 1024 * 1024 @@ -411,10 +411,10 @@ def make_cursor(self, path_or_fd): prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: - regions = self.MappedRegionListCls(path_or_fd) + regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return MemoryCursor(self, regions) + return SlidingCursor(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 48aeabb44..96ca4f882 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,6 +1,6 @@ from lib import TestBase, FileCreator -from smmap.mman import MappedMemoryManager +from smmap.mman import SlidingWindowMapManager from smmap.buf import * from random import randint @@ -9,8 +9,8 @@ import os -man_optimal = MappedMemoryManager() -man_worst_case = MappedMemoryManager( window_size=TestBase.k_window_test_size/100, +man_optimal = SlidingWindowMapManager() +man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, max_memory_size=TestBase.k_window_test_size/3, max_open_handles=15) @@ -21,10 +21,10 @@ def test_basics(self): # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size) # offset too large + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - buf = MappedMemoryBuffer() # can create uninitailized buffers + buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None # can call end access any time @@ -71,7 +71,7 @@ def test_basics(self): for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case')): - buf = MappedMemoryBuffer(manager.make_cursor(item)) + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e220f8bb2..dfe43e704 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MemoryCursor +from smmap.mman import SlidingCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -16,8 +16,8 @@ class TestMMan(TestBase): def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") - man = MappedMemoryManager() - ci = MemoryCursor(man) # invalid cursor + man = SlidingWindowMapManager() + ci = SlidingCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,10 +43,10 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - MemoryCursor(man)._destroy() + SlidingCursor(man)._destroy() def test_memory_manager(self): - man = MappedMemoryManager() + man = SlidingWindowMapManager() assert man.num_file_handles() == 0 assert man.num_open_files() == 0 assert man.window_size() > 0 @@ -82,7 +82,7 @@ def test_memman_operation(self): # small windows, a reasonable max memory. Not too many regions at once max_num_handles = 15 - man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) + man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) c = man.make_cursor(item) # still empty (more about that is tested in test_memory_manager() diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 4043cd83f..46de2ebaa 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -8,10 +8,10 @@ class TestMMan(TestBase): def test_window(self): - wl = MemoryWindow(0, 1) # left - wc = MemoryWindow(1, 1) # center - wc2 = MemoryWindow(10, 5) # another center - wr = MemoryWindow(8000, 50) # right + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right assert wl.ofs_end() == 1 assert wc.ofs_end() == 2 @@ -56,9 +56,9 @@ def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size / 2 rofs = align_to_mmap(4200, False) - rfull = MappedRegion(fc.path, 0, fc.size) - rhalfofs = MappedRegion(fc.path, rofs, fc.size) - rhalfsize = MappedRegion(fc.path, 0, half_size) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) # offsets assert rfull.ofs_begin() == 0 and rfull.size() == fc.size @@ -88,7 +88,7 @@ def test_region(self): assert rfull.usage_count() == 1 # window constructor - w = MemoryWindow.from_region(rfull) + w = MapWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): @@ -96,7 +96,7 @@ def test_region_list(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - ml = MappedRegionList(item) + ml = MapRegionList(item) assert ml.client_count() == 1 diff --git a/smmap/util.py b/smmap/util.py index 667777861..6ead86479 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -15,7 +15,7 @@ from sys import getrefcount __all__ = [ "align_to_mmap", "is_64_bit", - "MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"] + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -39,7 +39,7 @@ def is_64_bit(): #{ Utility Classes -class MemoryWindow(object): +class MapWindow(object): """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( 'ofs', # offset into the file in bytes @@ -51,7 +51,7 @@ def __init__(self, offset, size): self.size = size def __repr__(self): - return "MemoryWindow(%i, %i)" % (self.ofs, self.size) + return "MapWindow(%i, %i)" % (self.ofs, self.size) @classmethod def from_region(cls, region): @@ -84,7 +84,7 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class MappedRegion(object): +class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes :note: deallocates used region automatically on destruction""" __slots__ = [ @@ -145,7 +145,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): #END close file handle def __repr__(self): - return "MappedRegion<%i, %i>" % (self._b, self.size()) + return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface @@ -201,15 +201,15 @@ def includes_ofs(self, ofs): #} END interface -class MappedRegionList(list): - """List of MappedRegion instances associating a path with a list of regions.""" +class MapRegionList(list): + """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): - return super(MappedRegionList, cls).__new__(cls) + return super(MapRegionList, cls).__new__(cls) def __init__(self, path_or_fd): self._path_or_fd = path_or_fd From bc78951823623f9a529d0b515d85d6a0a1a1d8ac Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 15:31:07 +0200 Subject: [PATCH 0169/1790] moved code from cursor into manager, as it belongs there. This is a design error inherited from c++, but actually it makes overrides a bit harder, or lets say, less native, as the sliding mechanics where implemented in a class which is just the handle to a memory map in the end, which doesn't have to care about its allocation --- smmap/buf.py | 2 +- smmap/mman.py | 234 ++++++++++++++++++++-------------------- smmap/test/test_mman.py | 6 +- 3 files changed, 122 insertions(+), 120 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 94650a50c..741450303 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import SlidingCursor +from mman import MemoryCursor import sys diff --git a/smmap/mman.py b/smmap/mman.py index 312612298..16f878199 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -15,7 +15,7 @@ #}END utilities -class SlidingCursor(object): +class MemoryCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed. @@ -28,11 +28,6 @@ class SlidingCursor(object): '_size' # maximum size we should provide ) - #{ Configuration - MapWindowCls = MapWindow - MapRegionCls = MapRegion - #} END configuration - def __init__(self, manager = None, regions = None): self._manager = manager self._rlist = regions @@ -82,7 +77,7 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, flags = 0, _is_recursive=False): + def use_region(self, offset, size, flags = 0): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map @@ -104,115 +99,14 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # END handle existing region # END check existing region + # offset too large ? + if offset >= self._rlist.file_size(): + return self + #END handle offset + if need_region: - window_size = man._window_size - - # abort on offsets beyond our mapped file's size - currently we are invalid - if offset >= self.file_size(): - return self - # END handle offset too large - - # bisect to find an existing region. The c++ implementation cannot - # do that as it uses a linked list for regions. - existing_region = None - a = self._rlist - lo = 0 - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - ofs = a[mid]._b - if ofs <= offset: - if a[mid].includes_ofs(offset): - existing_region = a[mid] - break - #END have region - lo = mid+1 - else: - hi = mid - #END handle position - #END while bisecting - - if existing_region is None: - left = self.MapWindowCls(0, 0) - mid = self.MapWindowCls(offset, size) - right = self.MapWindowCls(self.file_size(), 0) - - # we want to honor the max memory size, and assure we have anough - # memory available - # Save calls ! - if self._manager._memory_size + window_size > self._manager._max_memory_size: - man._collect_lru_region(window_size) - #END handle collection - - # we assume the list remains sorted by offset - insert_pos = 0 - len_regions = len(a) - if len_regions == 1: - if a[0]._b <= offset: - insert_pos = 1 - #END maintain sort - else: - # find insert position - insert_pos = len_regions - for i, region in enumerate(a): - if region._b > offset: - insert_pos = i - break - #END if insert position is correct - #END for each region - # END obtain insert pos - - # adjust the actual offset and size values to create the largest - # possible mapping - if insert_pos == 0: - if len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side - else: - if insert_pos != len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - # END adjust right window - left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows - - mid.extend_left_to(left, window_size) - mid.extend_right_to(right, window_size) - mid.align() - - # it can happen that we align beyond the end of the file - if mid.ofs_end() > right.ofs: - mid.size = right.ofs - mid.ofs - #END readjust size - - # insert new region at the right offset to keep the order - try: - if man._handle_count >= man._max_handle_count: - raise Exception - #END assert own imposed max file handles - self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if _is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - man._collect_lru_region(0) - return self.use_region(offset, size, flags, True) - #END handle exceptions - - man._handle_count += 1 - man._memory_size += self._region.size() - a.insert(insert_pos, self._region) - else: - self._region = existing_region - #END need region handling - #END handle acquire region + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + #END need region handling self._region.increment_usage_count() self._ofs = offset - self._region._b @@ -301,6 +195,7 @@ def fd(self): #} END interface + class SlidingWindowMapManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. @@ -325,6 +220,8 @@ class SlidingWindowMapManager(object): #{ Configuration MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion #} END configuration _MB_in_bytes = 1024 * 1024 @@ -398,6 +295,111 @@ def _collect_lru_region(self, size): return num_found + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + #END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r + #{ Interface def make_cursor(self, path_or_fd): """:return: a cursor pointing to the given path or file descriptor. @@ -414,7 +416,7 @@ def make_cursor(self, path_or_fd): regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return SlidingCursor(self, regions) + return MemoryCursor(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index dfe43e704..79c6f9892 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import SlidingCursor +from smmap.mman import MemoryCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -17,7 +17,7 @@ def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") man = SlidingWindowMapManager() - ci = SlidingCursor(man) # invalid cursor + ci = MemoryCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,7 +43,7 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - SlidingCursor(man)._destroy() + MemoryCursor(man)._destroy() def test_memory_manager(self): man = SlidingWindowMapManager() From 03dd5ae25fe99b9b91212057ef57a09e40f23265 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 16:19:10 +0200 Subject: [PATCH 0170/1790] Changed design of memory managers to support different implementations. Currently there is a non-implemented static version, as well as the previous sliding window version. --- smmap/buf.py | 6 +- smmap/mman.py | 222 +++++++++++++++++++++++++++++--------------------- smmap/util.py | 4 +- 3 files changed, 134 insertions(+), 98 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 741450303..772a268e6 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,7 +10,11 @@ class SlidingWindowMapBuffer(object): memory of a mapped file. The mapping is controlled by the provided cursor. The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at the offset you used during initialization or begin_access""" + first byte at the offset you used during initialization or begin_access + + :note: Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" __slots__ = ( '_c', # our cursor '_size', # our supposed size diff --git a/smmap/mman.py b/smmap/mman.py index 16f878199..d799c77af 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -10,7 +10,7 @@ from weakref import ref import sys -__all__ = ["SlidingWindowMapManager"] +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] #{ Utilities #}END utilities @@ -125,11 +125,11 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset - to the window size. Please note that it might be smaller than you requested + to the window size. Please note that it might be smaller than you requested when calling use_region() :note: You can only obtain a buffer if this instance is_valid() ! :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) + return buffer(self._region.map(), self._ofs, self._size) def is_valid(self): """:return: True if we have a valid and usable region""" @@ -195,19 +195,17 @@ def fd(self): #} END interface +class StaticWindowMapManager(object): + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. -class SlidingWindowMapManager(object): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily - obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles - which result from each mmap call, the least recently used, and currently unused mapped regions - are unloaded automatically. + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. - :note: currently not thread-safe ! - :note: in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than - a safe amount of memory already, which would possibly cause memory allocations to fail as our address - space is full.""" + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + acomodate this fact""" __slots__ = [ '_fdict', # mapping of path -> MapRegionList @@ -222,11 +220,12 @@ class SlidingWindowMapManager(object): MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion + MemoryCursorCls = MemoryCursor #} END configuration _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. :param window_size: if 0, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size @@ -258,6 +257,8 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size + #{ Internal Methods + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full @@ -265,6 +266,117 @@ def _collect_lru_region(self, size): :raise RegionCollectionError: :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + raise NotImplementedError() + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + raise NotImplementedError() + + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """:return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + :note: if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + :note: Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.MemoryCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + :note: does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface + + + +class SlidingWindowMapManager(StaticWindowMapManager): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + :note: currently not thread-safe ! + :note: in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to 0""" + super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + + def _collect_lru_region(self, size): num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None @@ -296,10 +408,6 @@ def _collect_lru_region(self, size): return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, - see MapCursor.use_region. - :param a: A regions (a)rray - :return: The newly created region""" # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. r = None @@ -400,80 +508,4 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # END create new region return r - #{ Interface - def make_cursor(self, path_or_fd): - """:return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory - :note: if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - :note: Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" - regions = self._fdict.get(path_or_fd) - if regions is None: - regions = self.MapRegionListCls(path_or_fd) - self._fdict[path_or_fd] = regions - # END obtain region for path - return MemoryCursor(self, regions) - - def collect(self): - """Collect all available free-to-collect mapped regions - :return: Amount of freed handles""" - return self._collect_lru_region(0) - - def num_file_handles(self): - """:return: amount of file handles in use. Each mapped region uses one file handle""" - return self._handle_count - def num_open_files(self): - """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) - - def window_size(self): - """:return: size of each window when allocating new regions""" - return self._window_size - - def mapped_memory_size(self): - """:return: amount of bytes currently mapped in total""" - return self._memory_size - - def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" - return self._max_handle_count - - def max_mapped_memory_size(self): - """:return: maximum amount of memory we may allocate""" - return self._max_memory_size - - #} END interface - - #{ Special Purpose Interface - - def force_map_handle_removal_win(self, base_path): - """ONLY AVAILABLE ON WINDOWS - On windows removing files is not allowed if anybody still has it opened. - If this process is ourselves, and if the whole process uses this memory - manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to - allow the respective operation after all. - The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep - the cursors alive will not be using it anymore. They need to be recreated ! - :return: Amount of closed handles - :note: does nothing on non-windows platforms""" - if sys.platform != 'win32': - return - #END early bailout - - num_closed = 0 - for path, rlist in self._fdict.iteritems(): - if path.startswith(base_path): - for region in rlist: - region._mf.close() - num_closed += 1 - #END path matches - #END for each path - return num_closed - #} END special purpose interface diff --git a/smmap/util.py b/smmap/util.py index 6ead86479..21e285559 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -149,8 +149,8 @@ def __repr__(self): #{ Interface - def buffer(self): - """:return: a sliceable buffer which can be used to access the mapped memory""" + def map(self): + """:return: a memory map containing the memory""" return self._mf def ofs_begin(self): From 631b9ea2edfb005b1d48d55b35370f82ac2de196 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 17:47:48 +0200 Subject: [PATCH 0171/1790] Implemented static memory manager, for now without test --- smmap/buf.py | 2 +- smmap/mman.py | 98 ++++++++++++++++++++++++++++++++++++----- smmap/test/test_mman.py | 14 ++++-- 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 772a268e6..c4d252251 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import MemoryCursor +from mman import WindowCursor import sys diff --git a/smmap/mman.py b/smmap/mman.py index d799c77af..ba3a63f55 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -9,17 +9,23 @@ from exc import RegionCollectionError from weakref import ref import sys +from sys import getrefcount __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] #{ Utilities #}END utilities -class MemoryCursor(object): - """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed. + + +class WindowCursor(object): + """Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. - Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager""" + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + :note: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -130,7 +136,14 @@ def buffer(self): :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.map(), self._ofs, self._size) - + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None @@ -188,7 +201,7 @@ def fd(self): :note: it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): - return ValueError("File descriptor queried although mapping was generated from path") + raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() @@ -208,7 +221,7 @@ class StaticWindowMapManager(object): acomodate this fact""" __slots__ = [ - '_fdict', # mapping of path -> MapRegionList + '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate '_max_handle_count', # maximum amount of handles to keep open @@ -220,7 +233,7 @@ class StaticWindowMapManager(object): MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion - MemoryCursorCls = MemoryCursor + WindowCursorCls = WindowCursor #} END configuration _MB_in_bytes = 1024 * 1024 @@ -266,14 +279,77 @@ def _collect_lru_region(self, size): :raise RegionCollectionError: :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" - raise NotImplementedError() + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + for k, regions in self._fdict.iteritems(): + found_lonely_region = False + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): + # remove whole list + found_lonely_region = True + num_found += 1 + self._memory_size -= region.size() + self._handle_count -= 1 + self._fdict.pop(k) + + break + # END update lru_region + #END for each region + if found_lonely_region: + continue + # END skip iteration and restart + #END for each regions list + + # still here ? + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + #END while there is more memory to free + + return num_found + def _obtain_region(self, a, offset, size, flags, is_recursive): """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" - raise NotImplementedError() + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + # END handle array + + assert a.includes_ofs(offset) + assert a.includes_ofs(offset + size-1) + return r #}END internal methods @@ -293,7 +369,7 @@ def make_cursor(self, path_or_fd): regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return self.MemoryCursorCls(self, regions) + return self.WindowCursorCls(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 79c6f9892..e8266066c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MemoryCursor +from smmap.mman import WindowCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -17,7 +17,7 @@ def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") man = SlidingWindowMapManager() - ci = MemoryCursor(man) # invalid cursor + ci = WindowCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,7 +43,7 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - MemoryCursor(man)._destroy() + WindowCursor(man)._destroy() def test_memory_manager(self): man = SlidingWindowMapManager() @@ -65,11 +65,19 @@ def test_memory_manager(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): c = man.make_cursor(item) + assert c.path_or_fd() is item assert c.use_region(10, 10).is_valid() assert c.ofs_begin() == 10 assert c.size() == 10 assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error #END for each input + os.close(fd) def test_memman_operation(self): From e010084b0f40d7762f029a7ac4109c9bb99818a6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 18:28:37 +0200 Subject: [PATCH 0172/1790] test are running, once again, but not yet complete regarding the static manager --- smmap/mman.py | 92 ++++++++++++++--------------------------- smmap/test/test_mman.py | 71 +++++++++++++++++-------------- 2 files changed, 70 insertions(+), 93 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ba3a63f55..bdefe2e81 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -95,7 +95,8 @@ def use_region(self, offset, size, flags = 0): either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager - size = min(size, man.window_size()) # clamp size to window size + fsize = self._rlist.file_size() + size = min(size, man.window_size() or fsize) # clamp size to window size if self._region is not None: if self._region.includes_ofs(offset): @@ -106,7 +107,7 @@ def use_region(self, offset, size, flags = 0): # END check existing region # offset too large ? - if offset >= self._rlist.file_size(): + if offset >= fsize: return self #END handle offset @@ -238,10 +239,11 @@ class StaticWindowMapManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. - :param window_size: if 0, a default window size will be chosen depending on + :param window_size: if -1, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. @@ -254,7 +256,7 @@ def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handl self._memory_size = 0 self._handle_count = 0 - if window_size == 0: + if window_size < 0: coeff = 32 if is_64_bit(): coeff = 1024 @@ -281,43 +283,40 @@ def _collect_lru_region(self, size): :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): - for k, regions in self._fdict.iteritems(): - found_lonely_region = False + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and (lru_region is None or region._uc < lru_region._uc)): - # remove whole list - found_lonely_region = True - num_found += 1 - self._memory_size -= region.size() - self._handle_count -= 1 - self._fdict.pop(k) - - break + lru_region = region + lru_list = regions # END update lru_region #END for each region - if found_lonely_region: - continue - # END skip iteration and restart #END for each regions list - # still here ? - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary + if lru_region is None: + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 #END while there is more memory to free - return num_found - def _obtain_region(self, a, offset, size, flags, is_recursive): """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" - if self._memory_size + window_size > self._max_memory_size: - self._collect_lru_region(window_size) + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) #END handle collection r = None @@ -347,8 +346,8 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): self._memory_size += r.size() # END handle array - assert a.includes_ofs(offset) - assert a.includes_ofs(offset + size-1) + assert r.includes_ofs(offset) + assert r.includes_ofs(offset + size-1) return r #}END internal methods @@ -362,6 +361,8 @@ def make_cursor(self, path_or_fd): your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths are preferred unless you plan to keep the file descriptor open. + :note: file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. :note: Using file descriptors directly is faster once new windows are mapped as it prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) @@ -448,41 +449,10 @@ class SlidingWindowMapManager(StaticWindowMapManager): __slots__ = tuple() - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): - """Adjusts the default window size to 0""" + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - def _collect_lru_region(self, size): - num_found = 0 - while (size == 0) or (self._memory_size + size > self._max_memory_size): - lru_region = None - lru_list = None - for regions in self._fdict.itervalues(): - for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): - lru_region = region - lru_list = regions - # END update lru_region - #END for each region - #END for each regions list - - if lru_region is None: - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary - break - #END handle region not found - - num_found += 1 - del(lru_list[lru_list.index(lru_region)]) - self._memory_size -= lru_region.size() - self._handle_count -= 1 - #END while there is more memory to free - - return num_found - def _obtain_region(self, a, offset, size, flags, is_recursive): # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e8266066c..97866bbf9 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -46,40 +46,47 @@ def test_cursor(self): WindowCursor(man)._destroy() def test_memory_manager(self): - man = SlidingWindowMapManager() - assert man.num_file_handles() == 0 - assert man.num_open_files() == 0 - assert man.window_size() > 0 - assert man.mapped_memory_size() == 0 - assert man.max_mapped_memory_size() > 0 - - # collection doesn't raise in 'any' mode - man._collect_lru_region(0) - # doesn't raise if we are within the limit - man._collect_lru_region(10) - # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() - # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + #END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + # raises outside of limit + self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error + #END for each input + os.close(fd) + # END for each manager type - if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) - else: - self.failUnlessRaises(ValueError, c.fd) - #END handle value error - #END for each input - - os.close(fd) - def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") From 101a4d39108c4b4731760456523fd0f0d565b6f2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 19:50:43 +0200 Subject: [PATCH 0173/1790] Finally the mem manager and buffer tests run with the static one too --- smmap/mman.py | 18 ++-- smmap/test/test_buf.py | 6 +- smmap/test/test_mman.py | 216 ++++++++++++++++++++++------------------ 3 files changed, 131 insertions(+), 109 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index bdefe2e81..f1ce95d03 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,9 +4,9 @@ MapRegion, MapRegionList, is_64_bit, + align_to_mmap ) -from exc import RegionCollectionError from weakref import ref import sys from sys import getrefcount @@ -83,10 +83,10 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, flags = 0): + def use_region(self, offset, size = 0, flags = 0): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file - :param size: amount of bytes to map + :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. @@ -96,7 +96,7 @@ def use_region(self, offset, size, flags = 0): need_region = True man = self._manager fsize = self._rlist.file_size() - size = min(size, man.window_size() or fsize) # clamp size to window size + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size if self._region is not None: if self._region.includes_ofs(offset): @@ -246,6 +246,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, the manager will free as many handles as posisble""" @@ -278,8 +279,9 @@ def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region - :raise RegionCollectionError: :return: Amount of freed regions + :note: We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -297,9 +299,6 @@ def _collect_lru_region(self, size): #END for each regions list if lru_region is None: - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary break #END handle region not found @@ -344,10 +343,11 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): self._handle_count += 1 self._memory_size += r.size() + a.append(r) # END handle array assert r.includes_ofs(offset) - assert r.includes_ofs(offset + size-1) + #assert r.includes_ofs(offset+size-1) return r #}END internal methods diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 96ca4f882..d8b7fbcab 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,6 +1,6 @@ from lib import TestBase, FileCreator -from smmap.mman import SlidingWindowMapManager +from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager from smmap.buf import * from random import randint @@ -13,6 +13,7 @@ man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, max_memory_size=TestBase.k_window_test_size/3, max_open_handles=15) +static_man = StaticWindowMapManager() class TestBuf(TestBase): @@ -70,7 +71,8 @@ def test_basics(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case')): + (man_worst_case, 'worst case'), + (static_man, 'static optimial')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 97866bbf9..27be686ad 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -64,8 +64,9 @@ def test_memory_manager(self): man._collect_lru_region(0) # doesn't raise if we are within the limit man._collect_lru_region(10) - # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + # doesn't fail if we overallocate + assert man._collect_lru_region(sys.maxint) == 0 # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") @@ -92,103 +93,122 @@ def test_memman_operation(self): fc = FileCreator(self.k_window_test_size, "manager_operation_test") data = open(fc.path, 'rb').read() fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - max_num_handles = 15 - man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - assert c.size() == size - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) - - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # obtain second window, which spans the first part of the file - it is a still the same window - assert c.use_region(0, size-10).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == size-10 - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:size-10] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - size + overshoot - assert c.use_region(base_offset, size).is_valid() - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) + max_num_handles = 15 + #small_size = + for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) - # precondition - assert max_mapped_memory_size >= mapped_memory_size() - assert max_file_handles >= num_file_handles() + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() / 2 assert c.use_region(base_offset, size).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - #END for each item + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + + #assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + else: + assert rr().size() == fc.size + #END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + #END ignore static managers which only have one handle per file + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + #END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + #END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + # END for each manager type os.close(fd) From 82c97ea8f416bd99f501608b2aee8b7c4c5e78f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 20:12:40 +0200 Subject: [PATCH 0174/1790] Some more adjustments to make it work in all python versions --- smmap/mman.py | 2 +- smmap/util.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index f1ce95d03..fc6848413 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -136,7 +136,7 @@ def buffer(self): :note: You can only obtain a buffer if this instance is_valid() ! :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.map(), self._ofs, self._size) + return buffer(self._region.buffer(), self._ofs, self._size) def map(self): """ diff --git a/smmap/util.py b/smmap/util.py index 21e285559..80ba3c5b9 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -136,7 +136,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._size = len(self._mf) if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, size) + self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping finally: if isinstance(path_or_fd, basestring): @@ -148,7 +148,11 @@ def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface - + + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + def map(self): """:return: a memory map containing the memory""" return self._mf From a33e8d55d4d77d842edea94a78d801b23bb90294 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 21:59:22 +0200 Subject: [PATCH 0175/1790] Switched git db to the non-sliding version of the memory manager which is a good tradeoff between performance loss and resource handling --- gitdb/ext/smmap | 2 +- gitdb/pack.py | 76 +++++++++++++++++++++++-------------------------- gitdb/util.py | 13 +++++++++ 3 files changed, 50 insertions(+), 41 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 4466476cf..4cabaca18 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 4466476cf576cf5936a11524e67345a80e2ec5a9 +Subproject commit 4cabaca18cd30399288fc55863de412446ea51e7 diff --git a/gitdb/pack.py b/gitdb/pack.py index 7ae9786e6..0679a6ecf 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -10,10 +10,10 @@ ) from util import ( zlib, + mman, LazyMixin, unpack_from, bin_to_hex, - file_contents_ro_filepath, ) from fun import ( @@ -247,7 +247,7 @@ class PackIndexFile(LazyMixin): # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -261,22 +261,23 @@ def __init__(self, indexpath): def _set_cache_(self, attr): if attr == "_packfile_checksum": - self._packfile_checksum = self._data[-40:-20] + self._packfile_checksum = self._cursor.map()[-40:-20] elif attr == "_packfile_checksum": - self._packfile_checksum = self._data[-20:] - elif attr == "_data": + self._packfile_checksum = self._cursor.map()[-20:] + elif attr == "_cursor": # Note: We don't lock the file when reading as we cannot be sure # that we can actually write to the location - it could be a read-only # alternate for instance - self._data = file_contents_ro_filepath(self._indexpath) + self._cursor = mman.make_cursor(self._indexpath).use_region() else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties # CHECK VERSION - self._version = (self._data[:4] == self.index_v2_signature and 2) or 1 + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", self._data, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version @@ -297,16 +298,16 @@ def _set_cache_(self, attr): def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._data, 1024 + i*24) + (0, ) + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" - return unpack_from(">L", self._data, 1024 + i*24)[0] + return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] def _sha_v1(self, i): """see ``_sha_v2``""" base = 1024 + (i*24)+4 - return self._data[base:base+20] + return self._cursor.map()[base:base+20] def _crc_v1(self, i): """unsupported""" @@ -322,13 +323,13 @@ def _entry_v2(self, i): def _offset_v2(self, i): """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" - offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0] + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: - offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset return offset @@ -336,11 +337,11 @@ def _offset_v2(self, i): def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 - return self._data[base:base+20] + return self._cursor.map()[base:base+20] def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] #} END access V2 @@ -358,7 +359,7 @@ def _initialize(self): def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" - d = self._data + d = self._cursor.map() out = list() append = out.append for i in range(256): @@ -382,11 +383,11 @@ def path(self): def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" - return self._data[-40:-20] + return self._cursor.map()[-40:-20] def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" - return self._data[-20:] + return self._cursor.map()[-20:] def offsets(self): """:return: sequence of all offsets in the order in which they were written @@ -394,7 +395,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more if sys.byteorder == 'little': @@ -501,7 +502,7 @@ class PackFile(LazyMixin): for some reason - one clearly doesn't want to read 10GB at once in that case""" - __slots__ = ('_packpath', '_data', '_size', '_version') + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 @@ -513,26 +514,20 @@ def __init__(self, packpath): self._packpath = packpath def _set_cache_(self, attr): - if attr == '_data': - self._data = file_contents_ro_filepath(self._packpath) - - # read the header information - type_id, self._version, self._size = unpack_from(">LLL", self._data, 0) - - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - else: # must be '_size' or '_version' - # read header info - we do that just with a file stream - type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12)) - # END handle header + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - #END assert type id def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" - data = self._data + data = self._cursor.map() content_size = len(data) - self.footer_size cur_offset = start_offset or self.first_object_offset @@ -568,11 +563,11 @@ def data(self): """ :return: read-only data of this pack. It provides random access and usually is a memory map""" - return self._data + return self._cursor.map() def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._data[-20:] + return self._cursor.map()[-20:] def path(self): """:return: path to the packfile""" @@ -591,8 +586,9 @@ def collect_streams(self, offset): If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() + data = self._cursor.map() while True: - ostream = pack_object_at(self._data, offset, True)[1] + ostream = pack_object_at(data, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -614,14 +610,14 @@ def info(self, offset): :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, False)[1] + return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, True)[1] + return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """ @@ -704,7 +700,7 @@ def _object(self, sha, as_stream, index=-1): sha = self._index.sha(index) # END assure sha is present ( in output ) offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset)) if as_stream: if type_id not in delta_types: packstream = self._pack.stream(offset) diff --git a/gitdb/util.py b/gitdb/util.py index 4bb3c7352..4ce615585 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -23,6 +23,14 @@ # END try async zlib from async import ThreadPool +from smmap import ( + StaticWindowMapManager, + SlidingWindowMapBuffer + ) + +# initialize our global memory manager instance +# Use it to free cached (and unused) resources. +mman = StaticWindowMapManager() try: import hashlib @@ -180,6 +188,11 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): close(fd) # END assure file is closed +def sliding_ro_buffer(filepath, flags=0): + """:return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: From 34f01396b913220fe5b19e1f8e33f2d3f4ec2ce5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:05:36 +0200 Subject: [PATCH 0176/1790] Added changelog information --- doc/source/changes.rst | 5 +++++ gitdb/test/performance/test_pack.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 7b8ebecc6..999cc1309 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +***** +0.5.3 +***** +* Added support for smmap. SmartMMap allows resources to be managed and controlled. This brings the implementation closer to the way git handles memory maps, such that unused cached memory maps will automatically be freed once a resource limit is hit. The memory limit on 32 bit systems remains though as a sliding mmap implementation is not used for performance reasons. + ***** 0.5.2 ***** diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index da952b17a..20618024d 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -15,9 +15,11 @@ from time import time import random +from nose import SkipTest + class TestPackedDBPerformance(TestBigRepoR): - def _test_pack_random_access(self): + def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # sha lookup @@ -66,6 +68,7 @@ def _test_pack_random_access(self): print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) def test_correctness(self): + raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" From cb4059bccea6d01e41d63c66fda502823a35220f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:07:37 +0200 Subject: [PATCH 0177/1790] Bumped version info to 0.5.3 --- doc/source/conf.py | 4 ++-- setup.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 28deb3106..723a34503 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'GitDB' -copyright = u'2010, Sebastian Thiel' +copyright = u'2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.1' +release = '0.5.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3c6617422..86073971e 100755 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.2", + version = "0.5.3", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", @@ -80,7 +80,7 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.1)',), - install_requires='async >= 0.6.1', + requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), + install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""" ) From 09dd0eb9249493fc2d2897684035d753fc888acc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:22:42 +0200 Subject: [PATCH 0178/1790] A tiny win32 fix, once again --- smmap/test/test_util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 46de2ebaa..096c5f6df 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -71,9 +71,10 @@ def test_region(self): assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. - if sys.platform == 'win32': - assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0) - else: + # We only test on linux as it is inconsitent between the python versions + # as they use different mapping techniques to circumvent the missing offset + # argument of mmap. + if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) #END handle platforms From 84eedc5d1def7bfefefc729d09c39a6a9cde81f2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 14:54:17 +0200 Subject: [PATCH 0179/1790] Added option to help test cases to succeed on windows. Its for test systems only --- smmap/util.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index 80ba3c5b9..7ced28991 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -99,7 +99,13 @@ class MapRegion(object): if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot - + + #{ Configuration + # Used for testing only. If True, all data will be loaded into memory at once. + # This makes sure no file handles will remain open. + _test_read_into_memory = False + #} END configuration + def __init__(self, path_or_fd, ofs, size, flags = 0): """Initialize a region, allocate the memory map @@ -132,7 +138,13 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + if self._test_read_into_memory: + self._mf = self._read_into_memory(fd, ofs, actual_size) + else: + self._mf = mmap(fd, actual_size, **kwargs) + #END handle memory mode + self._size = len(self._mf) if self._need_compat_layer: @@ -144,6 +156,19 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): #END only close it if we opened it #END close file handle + def _read_into_memory(self, fd, offset, size): + """:return: string data as read from the given file descriptor, offset and size """ + os.lseek(fd, offset, os.SEEK_SET) + mf = '' + bytes_todo = size + while bytes_todo: + chunk = 1024*1024 + d = os.read(fd, chunk) + bytes_todo -= len(d) + mf += d + #END loop copy items + return mf + def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) @@ -204,7 +229,7 @@ def includes_ofs(self, ofs): #} END interface - + class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( From 65dacbbd74a46698932cccdcab54f7558d1da169 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 20:23:55 +0200 Subject: [PATCH 0180/1790] Fixed up configuration to create api documentation for the code. Improved some markup to be valid for sphinx. --- doc/source/conf.py | 2 +- smmap/mman.py | 5 +++-- smmap/util.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a0dac1166..90409a138 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../')) # -- General configuration ----------------------------------------------------- diff --git a/smmap/mman.py b/smmap/mman.py index c6efc9496..9629eca46 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -354,8 +354,9 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): #{ Interface def make_cursor(self, path_or_fd): - """:return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory :note: if a file descriptor is given, it is assumed to be open and valid, but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only diff --git a/smmap/util.py b/smmap/util.py index 7ced28991..07bdf7997 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -20,7 +20,9 @@ #{ Utilities def align_to_mmap(num, round_up): - """Align the given integer number to the closest page offset, which usually is 4096 bytes. + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" From 78cdb214fb6273169433fa662ad630afc26eb36a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 21:07:03 +0200 Subject: [PATCH 0181/1790] Wrote introduction, readme.rst now points to the introduction page to keep things consistent --- README.rst | 52 +--------------------------- doc/source/api.rst | 42 +++++++++++++++++++++++ doc/source/index.rst | 3 ++ doc/source/intro.rst | 82 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 51 deletions(-) mode change 100644 => 120000 README.rst create mode 100644 doc/source/api.rst create mode 100644 doc/source/intro.rst diff --git a/README.rst b/README.rst deleted file mode 100644 index 4c22eed2b..000000000 --- a/README.rst +++ /dev/null @@ -1,51 +0,0 @@ -#################### -Sliding MMap (smmap) -#################### -A straight forward implementation of a slidinging memory map. -The idea is that every access to a file goes through a memory map manager, which will on demand map a region of a file and provide a string-like object for reading. - -When reading from it, you will have to check whether you are still within your window boundary, and possibly obtain a new window as required. - -The great benefit of this system is that you can use it to map files of any size even on 32 bit systems. Additionally it will be able to close unused windows right away to return system resources. If there are multiple clients for the same file and location, the same window will be reused as well. - -As there is a global management facility, you are also able to forcibly free all open handles which is handy on windows, which would otherwise prevent the deletion of the involved files. - -For convenience, a stream class is provided which hides the usage of the memory manager behind a simple stream interface. - -************ -LIMITATIONS -************ -* The access is readonly by design. -* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0. - -************ -REQUIREMENTS -************ -* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function. - -******* -Install -******* -TODO - -****** -Source -****** -The source is available at git://github.com/Byron/smmap.git and can be cloned using:: - - git clone git://github.com/Byron/smmap.git - -************ -MAILING LIST -************ -http://groups.google.com/group/git-python - -************* -ISSUE TRACKER -************* -https://github.com/Byron/smmap/issues - -******* -LICENSE -******* -New BSD License diff --git a/README.rst b/README.rst new file mode 120000 index 000000000..7cafde78d --- /dev/null +++ b/README.rst @@ -0,0 +1 @@ +doc/source/intro.rst \ No newline at end of file diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..7e2854afa --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,42 @@ +.. _api-label: + +############# +API Reference +############# + +**************** +smmap.mman +**************** + +.. automodule:: smmap.mman + :members: + :undoc-members: + +**************** +smmap.buf +**************** + +.. automodule:: smmap.buf + :members: + :undoc-members: + +**************** +smmap.exc +**************** + +.. automodule:: smmap.exc + :members: + :undoc-members: + +**************** +smmap.util +**************** + +.. automodule:: smmap.util + :members: + :undoc-members: + + + + + diff --git a/doc/source/index.rst b/doc/source/index.rst index cb03044e0..d25ef8244 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,6 +12,9 @@ Contents: .. toctree:: :maxdepth: 2 + intro + tutorial + api changes Indices and tables diff --git a/doc/source/intro.rst b/doc/source/intro.rst new file mode 100644 index 000000000..b35e78569 --- /dev/null +++ b/doc/source/intro.rst @@ -0,0 +1,82 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +The interface also works around the missing offset parameter in python implementations up to python 2.5. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 2.4, 2.5 or 2.6 +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. +* It wasn't tested on python 2.7 and 3.x. + +############### +Getting Started +############### +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################ +Installing smmap +################ +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + + $ easy_install smmap + # or + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/Byron/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/Byron/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _pip: http://www.pip-installer.org/en/latest/ From a51b65d35d402791f774efe95d4b848cc524a403 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:48:55 +0200 Subject: [PATCH 0182/1790] Finished tutorial section, umproved capabilities of the buffer implementation to be more pythonic. Unfortunately, not all docs build yet because of some typical sphinx issue that results in an error which doesn't at all tell what the culprit actually is --- doc/source/intro.rst | 9 +-- doc/source/tutorial.rst | 118 ++++++++++++++++++++++++++++++++++++ smmap/buf.py | 10 +++ smmap/mman.py | 6 +- smmap/test/test_buf.py | 4 ++ smmap/test/test_tutorial.py | 83 +++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 doc/source/tutorial.rst create mode 100644 smmap/test/test_tutorial.py diff --git a/doc/source/intro.rst b/doc/source/intro.rst index b35e78569..30bff0ded 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -34,11 +34,6 @@ Limitations * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. * It wasn't tested on python 2.7 and 3.x. -############### -Getting Started -############### -It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. - ################ Installing smmap ################ @@ -53,7 +48,9 @@ As the command will install smmap in your respective python distribution, you wi If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: $ python setup.py install - + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + ################## Homepage and Links ################## diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 000000000..917b24594 --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,118 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying classes. + +****** +Design +****** +Per application, there is *MemoryManager* which is held as static instance and used throughout the application. It can be configured to keep your resources within certain limits. + +To access mapped regions, you require a cursor. Cursors point to exactly one file and serve as handles into it. As long as it exists, the respective memory region will remain available. + +For convenience, a buffer implementation is provided which handles cursors and resource allocation behind its simple buffer like interface. + +*************** +Memory Managers +*************** +There are two types of memory managers, one uses *static* windows, the other one uses *sliding* windows. A window is a region of a file mapped into memory. Although the names might be somewhat misleading as technically windows are always static, the *sliding* version will allocate relatively small windows whereas the *static* version will always map the whole file. + +The *static* manager does nothing more than keeping a client count on the respective memory maps which always map the whole file, which allows to make some assumptions that can lead to simplified data access and increased performance, but reduces the compatibility to 32 bit systems or giant files. + +The *sliding* memory manager therefore should be the default manager when preparing an application for handling huge amounts of data on 32 bit and 64 bit platforms:: + + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + mman.num_file_handles() + mman.mapped_memory_size() + # and many more ... + + +Cursors +******* +*Cursors* are handles that point onto a window, i.e. a region of a file mapped into memory. From them you may obtain a buffer through which the data of that window can actually be accessed:: + + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + +Now you would have to write your algorithms around this interface to properly slide through huge amounts of data. + +Alternatively you can use a convenience interface. + +******* +Buffers +******* +To make first use easier, at the expense of performance, there is a Buffer implementation which uses a cursor underneath. + +With it, you can access all data in a possibly huge file without having to take care of setting the cursor to different regions yourself:: + + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + +Disadvantages +************* +Buffers cannot be used in place of strings or maps, hence you have to slice them to have valid input for the sorts of struct and zlib. A slice means a lot of data handling overhead which makes buffers slower compared to using cursors directly. + diff --git a/smmap/buf.py b/smmap/buf.py index c4d252251..9b2402687 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -47,6 +47,8 @@ def __len__(self): def __getitem__(self, i): c = self._c assert c.is_valid() + if i < 0: + i = self._size + i if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage @@ -57,6 +59,12 @@ def __getslice__(self, i, j): # fast path, slice fully included - safes a concatenate operation and # should be the default assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxint: + j = self._size + if j < 0: + j = self._size + j if (c.ofs_begin() <= i) and (j < c.ofs_end()): b = c.ofs_begin() return c.buffer()[i-b:j-b] @@ -68,6 +76,7 @@ def __getslice__(self, i, j): md = str() while l: c.use_region(ofs, l) + assert c.is_valid() d = c.buffer()[:l] ofs += len(d) l -= len(d) @@ -102,6 +111,7 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): self._size = size #END set size return res + # END use our cursor return False def end_access(self): diff --git a/smmap/mman.py b/smmap/mman.py index 9629eca46..deba99809 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -11,17 +11,16 @@ import sys from sys import getrefcount -__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] #{ Utilities #}END utilities - class WindowCursor(object): """Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. - + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager :note: The current implementation is suited for static and sliding window managers, but it also means that it must be suited for the somewhat quite different sliding manager. It could be improved, but @@ -85,6 +84,7 @@ def assign(self, rhs): def use_region(self, offset = 0, size = 0, flags = 0): """Assure we point to a window which allows access to the given offset into the file + :param offset: absolute offset in bytes into the file :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d8b7fbcab..9881c6294 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -50,6 +50,10 @@ def test_basics(self): assert data[offset] == buf[0] assert data[offset:offset*2] == buf[0:offset] + # negative indices, partial slices + assert buf[-1] == buf[len(buf)-1] + assert buf[-10:] == buf[len(buf)-10:len(buf)] + # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py new file mode 100644 index 000000000..a9f4b1c08 --- /dev/null +++ b/smmap/test/test_tutorial.py @@ -0,0 +1,83 @@ +from lib import TestBase + +class TestTutorial(TestBase): + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + From ec511365bc641a320c66ce4e796918e58a1567c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:56:16 +0200 Subject: [PATCH 0183/1790] It turned out the :note: docstring was not supported. Now all documentation is being generated --- doc/source/api.rst | 24 ++++++++++++------------ smmap/mman.py | 39 ++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index 7e2854afa..cddd268c4 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -4,33 +4,33 @@ API Reference ############# -**************** -smmap.mman -**************** +*********************** +Mapped Memory Managers +*********************** .. automodule:: smmap.mman :members: :undoc-members: -**************** -smmap.buf -**************** +******* +Buffers +******* .. automodule:: smmap.buf :members: :undoc-members: -**************** -smmap.exc -**************** +********** +Exceptions +********** .. automodule:: smmap.exc :members: :undoc-members: -**************** -smmap.util -**************** +********* +Utilities +********* .. automodule:: smmap.util :members: diff --git a/smmap/mman.py b/smmap/mman.py index deba99809..15bb012b4 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -18,13 +18,15 @@ class WindowCursor(object): - """Pointer into the mapped region of the memory manager, keeping the map + """ + Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - :note: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but - I see no real need to do so.""" + + **Note**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -91,8 +93,9 @@ def use_region(self, offset = 0, size = 0, flags = 0): for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file - :note: The size actually mapped may be smaller than the given size. If that is the case, - either the file has reached its end, or the map was created between two existing regions""" + + **note**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager fsize = self._rlist.file_size() @@ -123,9 +126,10 @@ def use_region(self, offset = 0, size = 0, flags = 0): def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - :note: the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it - helps to free up resource more quickly""" + + **note** the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" self._region = None # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! @@ -133,9 +137,11 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - :note: You can only obtain a buffer if this instance is_valid() ! - :note: buffers should not be cached passed the duration of your access as it will - prevent resources from being freed even though they might not be accounted for anymore !""" + + **note** You can only obtain a buffer if this instance is_valid() ! + + **note** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) def map(self): @@ -155,7 +161,8 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - :note: only if is_valid() is True""" + + **note** only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): @@ -177,7 +184,8 @@ def region_ref(self): def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - :note: cursor must be valid for this to work""" + + **note** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) @@ -199,7 +207,8 @@ def path(self): def fd(self): """:return: file descriptor used to create the underlying mapping. - :note: it is not required to be valid anymore + + **note** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): raise ValueError("File descriptor queried although mapping was generated from path") From 59cd3da62b5038783deeb2883262682758ec1eec Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:58:44 +0200 Subject: [PATCH 0184/1790] Made README.rst a copy of intro.rst. unfortunately symlinks are not followed by github. This is a real issue to me ... --- README.rst | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) mode change 120000 => 100644 README.rst diff --git a/README.rst b/README.rst deleted file mode 120000 index 7cafde78d..000000000 --- a/README.rst +++ /dev/null @@ -1 +0,0 @@ -doc/source/intro.rst \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..30bff0ded --- /dev/null +++ b/README.rst @@ -0,0 +1,79 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +The interface also works around the missing offset parameter in python implementations up to python 2.5. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 2.4, 2.5 or 2.6 +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. +* It wasn't tested on python 2.7 and 3.x. + +################ +Installing smmap +################ +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + + $ easy_install smmap + # or + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/Byron/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/Byron/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _pip: http://www.pip-installer.org/en/latest/ From f097bd611a82289d6bb95074fdf596332cb1c980 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:10:08 +0200 Subject: [PATCH 0185/1790] Fixed wrong operating system fields --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e2bb622fd..2e97e59c6 100755 --- a/setup.py +++ b/setup.py @@ -41,8 +41,8 @@ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", - "Operating System :: Windows", - "Operating System :: OSX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", ], long_description=long_description, From cf297b7b81bc5f6011c49d818d776ed7915fa1ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:33:49 +0200 Subject: [PATCH 0186/1790] Removed possibly invalid documentation tags --- smmap/buf.py | 6 +++--- smmap/mman.py | 53 +++++++++++++++++++++++++++++---------------------- smmap/util.py | 3 ++- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 9b2402687..00ddbacd9 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -12,9 +12,9 @@ class SlidingWindowMapBuffer(object): The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access - :note: Although this type effectively hides the fact that there are mapped windows - underneath, it can unfortunately not be used in any non-pure python method which - needs a buffer or string""" + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" __slots__ = ( '_c', # our cursor '_size', # our supposed size diff --git a/smmap/mman.py b/smmap/mman.py index 15bb012b4..9b08ae969 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -24,7 +24,7 @@ class WindowCursor(object): Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - **Note**: The current implementation is suited for static and sliding window managers, but it also means + **Note:**: The current implementation is suited for static and sliding window managers, but it also means that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( @@ -94,7 +94,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file - **note**: The size actually mapped may be smaller than the given size. If that is the case, + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager @@ -127,7 +127,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - **note** the cursor unuses the region automatically upon destruction. It is recommended + **Note:** the cursor unuses the region automatically upon destruction. It is recommended to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None @@ -138,9 +138,9 @@ def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - **note** You can only obtain a buffer if this instance is_valid() ! + **Note:** You can only obtain a buffer if this instance is_valid() ! - **note** buffers should not be cached passed the duration of your access as it will + **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) @@ -162,7 +162,7 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - **note** only if is_valid() is True""" + **Note:** only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): @@ -185,7 +185,7 @@ def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - **note** cursor must be valid for this to work""" + **Note:** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) @@ -208,7 +208,7 @@ def path(self): def fd(self): """:return: file descriptor used to create the underlying mapping. - **note** it is not required to be valid anymore + **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): raise ValueError("File descriptor queried although mapping was generated from path") @@ -289,9 +289,11 @@ def _collect_lru_region(self, size): :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :return: Amount of freed regions - :note: We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. - :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None @@ -366,15 +368,18 @@ def make_cursor(self, path_or_fd): """ :return: a cursor pointing to the given path or file descriptor. It can be used to map new regions of the file into memory - :note: if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - :note: file descriptors are problematic as they are not necessarily unique, as two - different files opened and closed in succession might have the same file descriptor id. - :note: Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: regions = self.MapRegionListCls(path_or_fd) @@ -426,7 +431,8 @@ def force_map_handle_removal_win(self, base_path): This really may only be used if you know that the items which keep the cursors alive will not be using it anymore. They need to be recreated ! :return: Amount of closed handles - :note: does nothing on non-windows platforms""" + + **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return #END early bailout @@ -451,8 +457,9 @@ class SlidingWindowMapManager(StaticWindowMapManager): which result from each mmap call, the least recently used, and currently unused mapped regions are unloaded automatically. - :note: currently not thread-safe ! - :note: in the current implementation, we will automatically unload windows if we either cannot + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot create more memory maps (as the open file handles limit is hit) or if we have allocated more than a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" diff --git a/smmap/util.py b/smmap/util.py index 07bdf7997..b0fd83b3f 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -88,7 +88,8 @@ def extend_right_to(self, window, max_size): class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes - :note: deallocates used region automatically on destruction""" + + **Note:** deallocates used region automatically on destruction""" __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) From 4524faf0d0c5383268b134084954b34faeaa766d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:29:22 +0200 Subject: [PATCH 0187/1790] Fixed up docs for upcoming release. Bumped version to 0.5.3 --- Makefile | 2 +- gitdb/__init__.py | 7 +++++++ gitdb/db/base.py | 7 ++++--- gitdb/db/mem.py | 4 ++-- gitdb/db/pack.py | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 15 +++++++++------ gitdb/pack.py | 29 +++++++++++++++++------------ gitdb/stream.py | 6 +++--- gitdb/test/db/lib.py | 2 +- gitdb/util.py | 28 ++++++++++++++++++---------- setup.py | 9 +++++---- 12 files changed, 69 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index e65c55a6d..c6c159bdc 100644 --- a/Makefile +++ b/Makefile @@ -23,5 +23,5 @@ clean:: rm -f *.so coverage:: build - PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=dulwich --with-coverage --cover-erase --cover-inclusive gitdb + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=gitdb --with-coverage --cover-erase --cover-inclusive gitdb diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 775c969cf..91359105c 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -24,6 +24,13 @@ def _init_externals(): _init_externals() +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/gitdb" +version_info = (0, 5, 3) +__version__ = '.'.join(str(i) for i in version_info) + + # default imports from db import * from base import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2189d4193..984acafbf 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -72,7 +72,8 @@ def stream_async(self, reader): :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` :return: async.Reader yielding OStream|InvalidOStream instances in any order - :note: depending on the system configuration, it might not be possible to + + **Note:** depending on the system configuration, it might not be possible to read all OStreams at once. Instead, read them individually using reader.read(x) where x is small enough.""" # base implementation just uses the stream method repeatedly @@ -140,7 +141,7 @@ def store_async(self, reader): The same instances will be used in the output channel as were received in by the Reader. - :note:As some ODB implementations implement this operation atomic, they might + **Note:** As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" # base implementation uses store to perform the work @@ -158,7 +159,7 @@ def __init__(self, root_path): """Initialize this instance to look for its files at the given root path All subsequent operations will be relative to this path :raise InvalidDBRoot: - :note: The base will not perform any accessablity checking as the base + **Note:** The base will not perform any accessablity checking as the base might not yet be accessible, but become accessible before the first access.""" super(FileDBBase, self).__init__() diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 8012ad15e..5d76c83cc 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -33,7 +33,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): it to the actual physical storage, as it allows to query whether object already exists in the target storage before introducing actual IO - :note: memory is currently not threadsafe, hence the async methods cannot be used + **Note:** memory is currently not threadsafe, hence the async methods cannot be used for storing""" def __init__(self): @@ -92,7 +92,7 @@ def sha_iter(self): def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly - :note: the object will only be written if it did not exist in the target db + **Note:** the object will only be written if it did not exist in the target db :return: amount of streams actually copied into odb. If smaller than the amount of input shas, one or more objects did already exist in odb""" count = 0 diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eef3f712e..4c9d0b919 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -58,7 +58,7 @@ def _pack_info(self, sha): """:return: tuple(entity, index) for an item at the given sha :param sha: 20 or 40 byte sha :raise BadObject: - :note: This method is not thread-safe, but may be hit in multi-threaded + **Note:** This method is not thread-safe, but may be hit in multi-threaded operation. The worst thing that can happen though is a counter that was not incremented, or the list being in wrong order. So we safe the time for locking here, lets see how that goes""" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 84eedc5d1..f097bd611 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 84eedc5d1def7bfefefc729d09c39a6a9cde81f2 +Subproject commit f097bd611a82289d6bb95074fdf596332cb1c980 diff --git a/gitdb/fun.py b/gitdb/fun.py index 5bbe8efc3..66130ebee 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -138,7 +138,7 @@ def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than absofs. - :note: global method for performance only, it belongs to DeltaChunkList""" + **Note:** global method for performance only, it belongs to DeltaChunkList""" lo = 0 hi = len(dcl) while lo < hi: @@ -414,9 +414,11 @@ def pack_object_header_info(data): return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): - """:return: string defining the pack header comprised of the object type - and its incompressed size in bytes - :parmam obj_type: pack type_id of the object + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte hdr = str() # output string @@ -483,7 +485,7 @@ def stream_copy(read, write, size, chunk_size): Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size - :note: its much like stream_copy utility, but operates just using methods""" + **Note:** its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written # WRITE ALL DATA UP TO SIZE @@ -597,7 +599,8 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes - :note: transcribed to python from the similar routine in patch-delta.c""" + + **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf while i < delta_buf_size: diff --git a/gitdb/pack.py b/gitdb/pack.py index 0679a6ecf..d840441e9 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -173,7 +173,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): class IndexWriter(object): """Utility to cache index information, allowing to write all information later in one go to the given stream - :note: currently only writes v2 indices""" + **Note:** currently only writes v2 indices""" __slots__ = '_objs' def __init__(self): @@ -391,7 +391,8 @@ def indexfile_checksum(self): def offsets(self): """:return: sequence of all offsets in the order in which they were written - :note: return value can be random accessed, but may be immmutable""" + + **Note:** return value can be random accessed, but may be immmutable""" if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears @@ -497,10 +498,10 @@ class PackFile(LazyMixin): packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - :note: at some point, this might be implemented using streams as well, or - streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that - case""" + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' @@ -625,8 +626,9 @@ def stream_iter(self, start_offset=0): to access the data in the pack directly. :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. - :note: Iterating a pack directly is costly as the datastream has to be decompressed - to determine the bounds between the objects""" + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) #} END Read-Database like Interface @@ -902,9 +904,11 @@ def write_pack(cls, object_iter, pack_write, index_write=None, :param zlib_compression: the zlib compression level to use :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack and over all contents of the index. If index_write was None, index_binsha will be None - :note: The destination of the write functions is up to the user. It could - be a socket, or a file for instance - :note: writes only undeltified objects""" + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" objs = object_iter if not object_count: if not isinstance(object_iter, (tuple, list)): @@ -979,7 +983,8 @@ def create(cls, object_iter, base_dir, object_count = None, zlib_compression = z and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files :return: PackEntity instance initialized with the new pack - :note: for more information on the other parameters see the write_pack method""" + + **Note:** for more information on the other parameters see the write_pack method""" pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) diff --git a/gitdb/stream.py b/gitdb/stream.py index 8010a0551..632213c27 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -51,7 +51,7 @@ class DecompressMemMapReader(LazyMixin): To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. - :note: The chunk-size should be carefully selected as it will involve quite a bit + **Note:** The chunk-size should be carefully selected as it will involve quite a bit of string copying due to the way the zlib is implemented. Its very wasteful, hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here @@ -609,8 +609,8 @@ class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor - :note: operates on raw file descriptors - :note: for this to work, you have to use the close-method of this instance""" + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") # default exception diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 416c8c588..4af4483c7 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -65,7 +65,7 @@ def _assert_object_writing_simple(self, db): def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW - :note: requires write access to the database""" + **Note:** requires write access to the database""" # start in 'dry-run' mode, using a simple sha1 writer ostreams = (ZippedStoreShaWriter, None) for ostreamcls in ostreams: diff --git a/gitdb/util.py b/gitdb/util.py index 4ce615585..23784de3f 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -119,8 +119,9 @@ def __getslice__(self, start, end): #{ Routines def make_sha(source=''): - """A python2.4 workaround for the sha/hashlib module fiasco - :note: From the dulwich project """ + """A python2.4 workaround for the sha/hashlib module fiasco + + **Note** From the dulwich project """ try: return hashlib.sha1(source) except NameError: @@ -146,6 +147,7 @@ def allocate_memory(size): def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd + :param fd: file descriptor opened for reading :param stream: if False, random access is provided, otherwise the stream interface is provided. @@ -173,14 +175,16 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible + :return: random access compatible memory of the given filepath :param stream: see ``file_contents_ro`` :param allow_mmap: see ``file_contents_ro`` :param flags: additional flags to pass to os.open :raise OSError: If the file could not be opened - :note: for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object - databases anyway, and they use it with the help of the ``flags`` parameter""" + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: return file_contents_ro(fd, stream, allow_mmap) @@ -189,7 +193,8 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): # END assure file is closed def sliding_ro_buffer(filepath, flags=0): - """:return: a buffer compatible object which uses our mapped memory manager internally + """ + :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) @@ -254,7 +259,7 @@ class LockedFD(object): This type handles error correctly in that it will assure a consistent state on destruction. - :note: with this setup, parallel reading is not possible""" + **note** with this setup, parallel reading is not possible""" __slots__ = ("_filepath", '_fd', '_write') def __init__(self, filepath): @@ -283,7 +288,8 @@ def open(self, write=False, stream=False): and must not be closed directly :raise IOError: if the lock could not be retrieved :raise OSError: If the actual file could not be opened for reading - :note: must only be called once""" + + **note** must only be called once""" if self._write is not None: raise AssertionError("Called %s multiple times" % self.open) @@ -327,13 +333,15 @@ def commit(self): """When done writing, call this function to commit your changes into the actual file. The file descriptor will be closed, and the lockfile handled. - :note: can be called multiple times""" + + **Note** can be called multiple times""" self._end_writing(successful=True) def rollback(self): """Abort your operation without any changes. The file descriptor will be closed, and the lock released. - :note: can be called multiple times""" + + **Note** can be called multiple times""" self._end_writing(successful=False) def _end_writing(self, successful=True): diff --git a/setup.py b/setup.py index 86073971e..b5c8c7046 100755 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from distutils.command.build_ext import build_ext import os, sys +import gitdb as meta # wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None @@ -69,11 +70,11 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.3", + version = meta.__version__, description = "Git Object Database", - author = "Sebastian Thiel", - author_email = "byronimo@gmail.com", - url = "http://gitorious.org/git-python/gitdb", + author = meta.__author__, + author_email = meta.__contact__, + url = meta.__homepage__, packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':'gitdb'}, From a5ed410aa0d3bed587214c3c017af2916b740da1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 3 Jul 2011 13:39:19 +0200 Subject: [PATCH 0188/1790] removed test suite from being distributed. It didn't work properly anyway and I am not going to dig into the setup tools mess --- setup.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index b5c8c7046..7924b5a07 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from distutils.command.build_ext import build_ext import os, sys -import gitdb as meta # wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None @@ -68,15 +67,23 @@ def get_data_files(self): setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too +# NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot +# satisfy the dependencies at installation time, unfortunately, due to inherent limitations +# of distutils, which cannot install the prerequesites of a package before the acutal package. +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/gitdb" +version_info = (0, 5, 3) +__version__ = '.'.join(str(i) for i in version_info) + setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = meta.__version__, + version = __version__, description = "Git Object Database", - author = meta.__author__, - author_email = meta.__contact__, - url = meta.__homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), - package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + author = __author__, + author_email = __contact__, + url = __homepage__, + packages = ('gitdb', 'gitdb.db'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", From aea587d9b414d7f150922c2923a1b9394d0d0543 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 08:58:22 +0200 Subject: [PATCH 0189/1790] Added license info for packs --- LICENSE | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/LICENSE b/LICENSE index be11e73c1..0d6fe8bdb 100644 --- a/LICENSE +++ b/LICENSE @@ -28,3 +28,15 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Additional Licenses +------------------- +The files at +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx +and +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack +are licensed under GNU GPL as part of the git source repository, +see http://en.wikipedia.org/wiki/Git_%28software%29 for more information. + +They are not required for the actual operation, which is why they are not found +in the distribution package. From 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:36:30 +0200 Subject: [PATCH 0190/1790] Fixed possible bug as a method was called using an old signature. Apparently this code branch never ran in the tests --- smmap/mman.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9b08ae969..ef9d43df0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -68,7 +68,7 @@ def _copy_from(self, rhs): self._size = rhs._size if self._region is not None: - self._region.increment_usage_count(1) + self._region.increment_usage_count() # END handle regions def __copy__(self): @@ -358,7 +358,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # END handle array assert r.includes_ofs(offset) - #assert r.includes_ofs(offset+size-1) return r #}END internal methods From 0e64168dd3f43b02857e60183d40c86480f01dc7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 15:03:16 +0200 Subject: [PATCH 0191/1790] pack: updated to use its cursor properly, which will be required if huge packs should be handled. This reduces performance as each access requires the windows to be checked/adjusted, but that is how it is. This should be circumvented using other backends, like the one of the gitcmd or libgit2. Default is now the sliding memory map manager --- gitdb/pack.py | 32 +++++++++++++++++++------------- gitdb/util.py | 3 ++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index d840441e9..c6d1cc313 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -73,7 +73,7 @@ #{ Utilities -def pack_object_at(data, offset, as_stream): +def pack_object_at(cursor, offset, as_stream): """ :return: Tuple(abs_data_offset, PackInfo|PackStream) an object of the correct type according to the type_id of the object. @@ -83,7 +83,7 @@ def pack_object_at(data, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - data = buffer(data, offset) + data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None @@ -269,6 +269,10 @@ def _set_cache_(self, attr): # that we can actually write to the location - it could be a read-only # alternate for instance self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) + #END assert window size else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -528,13 +532,13 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" - data = self._cursor.map() - content_size = len(data) - self.footer_size + c = self._cursor + content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset null = NullStream() while cur_offset < content_size: - data_offset, ostream = pack_object_at(data, cur_offset, True) + data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset @@ -563,12 +567,14 @@ def version(self): def data(self): """ :return: read-only data of this pack. It provides random access and usually - is a memory map""" - return self._cursor.map() + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.map()[-20:] + return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] def path(self): """:return: path to the packfile""" @@ -587,9 +593,9 @@ def collect_streams(self, offset): If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() - data = self._cursor.map() + c = self._cursor while True: - ostream = pack_object_at(data, offset, True)[1] + ostream = pack_object_at(c, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -611,14 +617,14 @@ def info(self, offset): :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1] + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1] + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """ @@ -702,7 +708,7 @@ def _object(self, sha, as_stream, index=-1): sha = self._index.sha(index) # END assure sha is present ( in output ) offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) if as_stream: if type_id not in delta_types: packstream = self._pack.stream(offset) diff --git a/gitdb/util.py b/gitdb/util.py index 23784de3f..e96c133e8 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -25,12 +25,13 @@ from async import ThreadPool from smmap import ( StaticWindowMapManager, + SlidingWindowMapManager, SlidingWindowMapBuffer ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = StaticWindowMapManager() +mman = SlidingWindowMapManager() try: import hashlib From ef5dc3d968b3aeed16a02ec705f89b72ad46fa84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:40:54 +0200 Subject: [PATCH 0192/1790] Optimized test_pack_streaming not to cache the objects anymore. Instead an iterator is provided which does the job. Previously it would easily use 750 MB of ram to keep all the associated objects, more than 350k. Still a lot of memory for just 350k objects, but its python after all --- gitdb/ext/smmap | 2 +- gitdb/test/performance/test_pack_streaming.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f097bd611..9c3eb3daf 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f097bd611a82289d6bb95074fdf596332cb1c980 +Subproject commit 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 795ed1e26..3c40ed0fb 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -40,10 +40,9 @@ def test_pack_writing(self): count = 0 total_size = 0 st = time() - objs = list() for sha in pdb.sha_iter(): count += 1 - objs.append(pdb.stream(sha)) + pdb.stream(sha) if count == ni: break #END gather objects for pack-writing @@ -51,7 +50,7 @@ def test_pack_writing(self): print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) st = time() - PackEntity.write_pack(objs, ostream.write) + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) From a4deb8461e6fbca0306a24f22b0c494679ad4757 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:45:39 +0200 Subject: [PATCH 0193/1790] wrote change log for next release. Choosing memory manager type based on the actual python version for best efficiency --- doc/source/changes.rst | 5 +++++ gitdb/util.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 999cc1309..839bf16a8 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +***** +0.5.4 +***** +* Adjusted implementation to use the SlidingMemoryManager by default in python 2.6 for efficiency reasons. In Python 2.4, the StaticMemoryManager will be used instead. + ***** 0.5.3 ***** diff --git a/gitdb/util.py b/gitdb/util.py index e96c133e8..013f5fc78 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -31,7 +31,11 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = SlidingWindowMapManager() +if sys.version_info[1] < 6: + mman = StaticWindowMapManager() +else: + mman = SlidingWindowMapManager() +#END handle mman try: import hashlib From d13e3aeb168645965720ddec1f469e05771563ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:55:32 +0200 Subject: [PATCH 0194/1790] updated changelog, bumped version --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ee17e0af0..03148fb31 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +********** +v0.8.1 +********** +- A single bugfix + + ********** v0.8.0 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 769858fa5..ae6e72eba 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 0) +version_info = (0, 8, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 656a2e0b4da7d60ac638d1615751a89efb3a4eee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 17:00:27 +0200 Subject: [PATCH 0195/1790] bumped version to 0.5.4 --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 91359105c..800b292da 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 3) +version_info = (0, 5, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 9c3eb3daf..d13e3aeb1 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b +Subproject commit d13e3aeb168645965720ddec1f469e05771563ef diff --git a/setup.py b/setup.py index 7924b5a07..62bc6d007 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 3) +version_info = (0, 5, 4) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, From e2b170a80462255dcc2003380db3547f55d8f14d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 8 Jul 2011 08:13:00 -0400 Subject: [PATCH 0196/1790] Workaround for #1 --- smmap/mman.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ef9d43df0..f5b7efbd1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -52,11 +52,14 @@ def _destroy(self): if self._rlist is not None: # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: - # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path_or_fd()) - #END remove regions list from manager + try: + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except TypeError: + pass #END handle regions def _copy_from(self, rhs): From bdc1258abb48328a389720df2ffc404692add426 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 29 Aug 2011 21:45:59 +0200 Subject: [PATCH 0197/1790] Added LICENSE file containing a copy of (new)BSD --- LICENSE | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..710010f1f --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the async project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + From 40fd4f31ab594dcfe049032c62ec61d2f0c3e492 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Jan 2012 23:09:53 +0100 Subject: [PATCH 0198/1790] Added some more in-code comments to clarify why that exception is caught --- smmap/mman.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/smmap/mman.py b/smmap/mman.py index f5b7efbd1..7b0984358 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -59,7 +59,12 @@ def _destroy(self): self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager except TypeError: + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount alltogether. pass + #END exception handling #END handle regions def _copy_from(self, rhs): From 360a8956fe73a0a96315e946f52737569d990369 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Jan 2012 23:12:02 +0100 Subject: [PATCH 0199/1790] Bumped version to 0.8.2 --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index ae6e72eba..a10cd5c99 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 1) +version_info = (0, 8, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From e96d2c381ef06667726eb745c67357de9d2a88fb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Jul 2012 20:56:03 +0200 Subject: [PATCH 0200/1790] Submodules now use the http protocol to facilitate checkout in corporate networks --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5dfd8e993..062ec5ba1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = git://github.com/gitpython-developers/async.git + url = http://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap - url = git://github.com/Byron/smmap.git + url = http://github.com/Byron/smmap.git From ec6998f503e4619cd6bdecbbf372552ea126900a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Jul 2012 21:12:03 +0200 Subject: [PATCH 0201/1790] Updated submodules to latest version --- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/ext/async b/gitdb/ext/async index 10310824c..039c1d5c2 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 10310824c001deab8fea85b88ebda0696f964b3e +Subproject commit 039c1d5c26bc2ceaa9e55082efae2068d9873e45 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index d13e3aeb1..360a8956f 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit d13e3aeb168645965720ddec1f469e05771563ef +Subproject commit 360a8956fe73a0a96315e946f52737569d990369 From 0328caa516fffdbb5f28fd59798a9775aa2b05f5 Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 2 Nov 2012 10:43:48 +1100 Subject: [PATCH 0202/1790] Update gitmodules to point to the https location of the git repositories. Signed-off-by: David --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 062ec5ba1..978105388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = http://github.com/gitpython-developers/async.git + url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap - url = http://github.com/Byron/smmap.git + url = https://github.com/Byron/smmap.git From 1b3ab5598e93369282502d049d64cb2ca12839cb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Feb 2014 20:50:30 +0100 Subject: [PATCH 0203/1790] tabs to spaces --- setup.py | 0 smmap/buf.py | 246 ++++---- smmap/exc.py | 6 +- smmap/mman.py | 1122 +++++++++++++++++------------------ smmap/test/lib.py | 96 +-- smmap/test/test_buf.py | 204 +++---- smmap/test/test_mman.py | 400 ++++++------- smmap/test/test_tutorial.py | 160 ++--- smmap/test/test_util.py | 212 +++---- smmap/util.py | 464 +++++++-------- 10 files changed, 1455 insertions(+), 1455 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/smmap/buf.py b/smmap/buf.py index 00ddbacd9..255c6b54d 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -6,129 +6,129 @@ __all__ = ["SlidingWindowMapBuffer"] class SlidingWindowMapBuffer(object): - """A buffer like object which allows direct byte-wise object and slicing into - memory of a mapped file. The mapping is controlled by the provided cursor. - - The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at the offset you used during initialization or begin_access - - **Note:** Although this type effectively hides the fact that there are mapped windows - underneath, it can unfortunately not be used in any non-pure python method which - needs a buffer or string""" - __slots__ = ( - '_c', # our cursor - '_size', # our supposed size - ) - - - def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): - """Initalize the instance to operate on the given cursor. - :param cursor: if not None, the associated cursor to the file you want to access - If None, you have call begin_access before using the buffer and provide a cursor - :param offset: absolute offset in bytes - :param size: the total size of the mapping. Defaults to the maximum possible size - From that point on, the __len__ of the buffer will be the given size or the file size. - If the size is larger than the mappable area, you can only access the actually available - area, although the length of the buffer is reported to be your given size. - Hence it is in your own interest to provide a proper size ! - :param flags: Additional flags to be passed to os.open - :raise ValueError: if the buffer could not achieve a valid state""" - self._c = cursor - if cursor and not self.begin_access(cursor, offset, size, flags): - raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") - # END handle offset + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by the provided cursor. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at the offset you used during initialization or begin_access + + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) + + + def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset - def __del__(self): - self.end_access() - - def __len__(self): - return self._size - - def __getitem__(self, i): - c = self._c - assert c.is_valid() - if i < 0: - i = self._size + i - if not c.includes_ofs(i): - c.use_region(i, 1) - # END handle region usage - return c.buffer()[i-c.ofs_begin()] - - def __getslice__(self, i, j): - c = self._c - # fast path, slice fully included - safes a concatenate operation and - # should be the default - assert c.is_valid() - if i < 0: - i = self._size + i - if j == sys.maxint: - j = self._size - if j < 0: - j = self._size + j - if (c.ofs_begin() <= i) and (j < c.ofs_end()): - b = c.ofs_begin() - return c.buffer()[i-b:j-b] - else: - l = j-i # total length - ofs = i - # Keeping tokens in a list could possible be faster, but the list - # overhead outweighs the benefits (tested) ! - md = str() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - md += d - #END while there are bytes to read - return md - # END fast or slow path - #{ Interface - - def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): - """Call this before the first use of this instance. The method was already - called by the constructor in case sufficient information was provided. - - For more information no the parameters, see the __init__ method - :param path: if cursor is None the existing one will be used. - :return: True if the buffer can be used""" - if cursor: - self._c = cursor - #END update our cursor - - # reuse existing cursors if possible - if self._c is not None and self._c.is_associated(): - res = self._c.use_region(offset, size, flags).is_valid() - if res: - # if given size is too large or default, we computer a proper size - # If its smaller, we assume the combination between offset and size - # as chosen by the user is correct and use it ! - # If not, the user is in trouble. - if size > self._c.file_size(): - size = self._c.file_size() - offset - #END handle size - self._size = size - #END set size - return res - # END use our cursor - return False - - def end_access(self): - """Call this method once you are done using the instance. It is automatically - called on destruction, and should be called just in time to allow system - resources to be freed. - - Once you called end_access, you must call begin access before reusing this instance!""" - self._size = 0 - if self._c is not None: - self._c.unuse_region() - #END unuse region - - def cursor(self): - """:return: the currently set cursor which provides access to the data""" - return self._c - - #}END interface + def __del__(self): + self.end_access() + + def __len__(self): + return self._size + + def __getitem__(self, i): + c = self._c + assert c.is_valid() + if i < 0: + i = self._size + i + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + return c.buffer()[i-c.ofs_begin()] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxint: + j = self._size + if j < 0: + j = self._size + j + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i-b:j-b] + else: + l = j-i # total length + ofs = i + # Keeping tokens in a list could possible be faster, but the list + # overhead outweighs the benefits (tested) ! + md = str() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + md += d + #END while there are bytes to read + return md + # END fast or slow path + #{ Interface + + def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if cursor is None the existing one will be used. + :return: True if the buffer can be used""" + if cursor: + self._c = cursor + #END update our cursor + + # reuse existing cursors if possible + if self._c is not None and self._c.is_associated(): + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + #END handle size + self._size = size + #END set size + return res + # END use our cursor + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 + if self._c is not None: + self._c.unuse_region() + #END unuse region + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/exc.py b/smmap/exc.py index a090d24d5..f0ed7dcd8 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -1,7 +1,7 @@ """Module with system exceptions""" class MemoryManagerError(Exception): - """Base class for all exceptions thrown by the memory manager""" - + """Base class for all exceptions thrown by the memory manager""" + class RegionCollectionError(MemoryManagerError): - """Thrown if a memory region could not be collected, or if no region for collection was found""" + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index 7b0984358..97c42c5bb 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,11 +1,11 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" from util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - align_to_mmap - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + align_to_mmap + ) from weakref import ref import sys @@ -18,564 +18,564 @@ class WindowCursor(object): - """ - Pointer into the mapped region of the memory manager, keeping the map - alive until it is destroyed and no other client uses it. + """ + Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. - Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - - **Note:**: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but - I see no real need to do so.""" - __slots__ = ( - '_manager', # the manger keeping all file regions - '_rlist', # a regions list with regions for our file - '_region', # our current region or None - '_ofs', # relative offset from the actually mapped area to our start area - '_size' # maximum size we should provide - ) - - def __init__(self, manager = None, regions = None): - self._manager = manager - self._rlist = regions - self._region = None - self._ofs = 0 - self._size = 0 - - def __del__(self): - self._destroy() - - def _destroy(self): - """Destruction code to decrement counters""" - self.unuse_region() - - if self._rlist is not None: - # Actual client count, which doesn't include the reference kept by the manager, nor ours - # as we are about to be deleted - try: - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: - # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path_or_fd()) - # END remove regions list from manager - except TypeError: - # sometimes, during shutdown, getrefcount is None. Its possible - # to re-import it, however, its probably better to just ignore - # this python problem (for now). - # The next step is to get rid of the error prone getrefcount alltogether. - pass - #END exception handling - #END handle regions - - def _copy_from(self, rhs): - """Copy all data from rhs into this instance, handles usage count""" - self._manager = rhs._manager - self._rlist = rhs._rlist - self._region = rhs._region - self._ofs = rhs._ofs - self._size = rhs._size - - if self._region is not None: - self._region.increment_usage_count() - # END handle regions - - def __copy__(self): - """copy module interface""" - cpy = type(self)() - cpy._copy_from(self) - return cpy - - #{ Interface - def assign(self, rhs): - """Assign rhs to this instance. This is required in order to get a real copy. - Alternativly, you can copy an existing instance using the copy module""" - self._destroy() - self._copy_from(rhs) - - def use_region(self, offset = 0, size = 0, flags = 0): - """Assure we point to a window which allows access to the given offset into the file - - :param offset: absolute offset in bytes into the file - :param size: amount of bytes to map. If 0, all available bytes will be mapped - :param flags: additional flags to be given to os.open in case a file handle is initially opened - for mapping. Has no effect if a region can actually be reused. - :return: this instance - it should be queried for whether it points to a valid memory region. - This is not the case if the mapping failed becaues we reached the end of the file - - **Note:**: The size actually mapped may be smaller than the given size. If that is the case, - either the file has reached its end, or the map was created between two existing regions""" - need_region = True - man = self._manager - fsize = self._rlist.file_size() - size = min(size or fsize, man.window_size() or fsize) # clamp size to window size - - if self._region is not None: - if self._region.includes_ofs(offset): - need_region = False - else: - self.unuse_region() - # END handle existing region - # END check existing region - - # offset too large ? - if offset >= fsize: - return self - #END handle offset - - if need_region: - self._region = man._obtain_region(self._rlist, offset, size, flags, False) - #END need region handling - - self._region.increment_usage_count() - self._ofs = offset - self._region._b - self._size = min(size, self._region.ofs_end() - offset) - - return self - - def unuse_region(self): - """Unuse the ucrrent region. Does nothing if we have no current region - - **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it - helps to free up resource more quickly""" - self._region = None - # note: should reset ofs and size, but we spare that for performance. Its not - # allowed to query information if we are not valid ! + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" + __slots__ = ( + '_manager', # the manger keeping all file regions + '_rlist', # a regions list with regions for our file + '_region', # our current region or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) + + def __init__(self, manager = None, regions = None): + self._manager = manager + self._rlist = regions + self._region = None + self._ofs = 0 + self._size = 0 + + def __del__(self): + self._destroy() + + def _destroy(self): + """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + try: + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except TypeError: + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount alltogether. + pass + #END exception handling + #END handle regions + + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = rhs._rlist + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + if self._region is not None: + self._region.increment_usage_count() + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy + + #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternativly, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset = 0, size = 0, flags = 0): + """Assure we point to a window which allows access to the given offset into the file + + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map. If 0, all available bytes will be mapped + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed becaues we reached the end of the file + + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + fsize = self._rlist.file_size() + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size + + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + # offset too large ? + if offset >= fsize: + return self + #END handle offset + + if need_region: + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + #END need region handling + + self._region.increment_usage_count() + self._ofs = offset - self._region._b + self._size = min(size, self._region.ofs_end() - offset) + + return self + + def unuse_region(self): + """Unuse the ucrrent region. Does nothing if we have no current region + + **Note:** the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! - def buffer(self): - """Return a buffer object which allows access to our memory region from our offset - to the window size. Please note that it might be smaller than you requested when calling use_region() - - **Note:** You can only obtain a buffer if this instance is_valid() ! - - **Note:** buffers should not be cached passed the duration of your access as it will - prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) - - def map(self): - """ - :return: the underlying raw memory map. Please not that the offset and size is likely to be different - to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole - file in case of StaticWindowMapManager""" - return self._region.map() - - def is_valid(self): - """:return: True if we have a valid and usable region""" - return self._region is not None - - def is_associated(self): - """:return: True if we are associated with a specific file already""" - return self._rlist is not None - - def ofs_begin(self): - """:return: offset to the first byte pointed to by our cursor - - **Note:** only if is_valid() is True""" - return self._region._b + self._ofs - - def ofs_end(self): - """:return: offset to one past the last available byte""" - # unroll method calls for performance ! - return self._region._b + self._ofs + self._size - - def size(self): - """:return: amount of bytes we point to""" - return self._size - - def region_ref(self): - """:return: weak ref to our mapped region. - :raise AssertionError: if we have no current region. This is only useful for debugging""" - if self._region is None: - raise AssertionError("region not set") - return ref(self._region) - - def includes_ofs(self, ofs): - """:return: True if the given absolute offset is contained in the cursors - current region - - **Note:** cursor must be valid for this to work""" - # unroll methods - return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) - - def file_size(self): - """:return: size of the underlying file""" - return self._rlist.file_size() - - def path_or_fd(self): - """:return: path or file decriptor of the underlying mapped file""" - return self._rlist.path_or_fd() + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested when calling use_region() + + **Note:** You can only obtain a buffer if this instance is_valid() ! + + **Note:** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" + return buffer(self._region.buffer(), self._ofs, self._size) + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor + + **Note:** only if is_valid() is True""" + return self._region._b + self._ofs + + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region._b + self._ofs + self._size + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region_ref(self): + """:return: weak ref to our mapped region. + :raise AssertionError: if we have no current region. This is only useful for debugging""" + if self._region is None: + raise AssertionError("region not set") + return ref(self._region) + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + + **Note:** cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path_or_fd(self): + """:return: path or file decriptor of the underlying mapped file""" + return self._rlist.path_or_fd() - def path(self): - """:return: path of the underlying mapped file - :raise ValueError: if attached path is not a path""" - if isinstance(self._rlist.path_or_fd(), int): - raise ValueError("Path queried although mapping was applied to a file descriptor") - # END handle type - return self._rlist.path_or_fd() - - def fd(self): - """:return: file descriptor used to create the underlying mapping. - - **Note:** it is not required to be valid anymore - :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), basestring): - raise ValueError("File descriptor queried although mapping was generated from path") - #END handle type - return self._rlist.path_or_fd() - - #} END interface - - + def path(self): + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + + **Note:** it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), basestring): + raise ValueError("File descriptor queried although mapping was generated from path") + #END handle type + return self._rlist.path_or_fd() + + #} END interface + + class StaticWindowMapManager(object): - """Provides a manager which will produce single size cursors that are allowed - to always map the whole file. - - Clients must be written to specifically know that they are accessing their data - through a StaticWindowMapManager, as they otherwise have to deal with their window size. - - These clients would have to use a SlidingWindowMapBuffer to hide this fact. - - This type will always use a maximum window size, and optimize certain methods to - acomodate this fact""" - - __slots__ = [ - '_fdict', # mapping of path -> StorageHelper (of some kind - '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount ofmemory we may allocate - '_max_handle_count', # maximum amount of handles to keep open - '_memory_size', # currently allocated memory size - '_handle_count', # amount of currently allocated file handles - ] - - #{ Configuration - MapRegionListCls = MapRegionList - MapWindowCls = MapWindow - MapRegionCls = MapRegion - WindowCursorCls = WindowCursor - #} END configuration - - _MB_in_bytes = 1024 * 1024 - - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): - """initialize the manager with the given parameters. - :param window_size: if -1, a default window size will be chosen depending on - the operating system's architechture. It will internally be quantified to a multiple of the page size - If 0, the window may have any size, which basically results in mapping the whole file at one - :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. - If 0, a viable default iwll be set dependning on the system's architecture. - It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate - :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, - the manager will free as many handles as posisble""" - self._fdict = dict() - self._window_size = window_size - self._max_memory_size = max_memory_size - self._max_handle_count = max_open_handles - self._memory_size = 0 - self._handle_count = 0 - - if window_size < 0: - coeff = 32 - if is_64_bit(): - coeff = 1024 - #END handle arch - self._window_size = coeff * self._MB_in_bytes - # END handle max window size - - if max_memory_size == 0: - coeff = 512 - if is_64_bit(): - coeff = 8192 - #END handle arch - self._max_memory_size = coeff * self._MB_in_bytes - #END handle max memory size - - #{ Internal Methods - - def _collect_lru_region(self, size): - """Unmap the region which was least-recently used and has no client - :param size: size of the region we want to map next (assuming its not already mapped partially or full - if 0, we try to free any available region - :return: Amount of freed regions - - **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. - - **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" - num_found = 0 - while (size == 0) or (self._memory_size + size > self._max_memory_size): - lru_region = None - lru_list = None - for regions in self._fdict.itervalues(): - for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): - lru_region = region - lru_list = regions - # END update lru_region - #END for each region - #END for each regions list - - if lru_region is None: - break - #END handle region not found - - num_found += 1 - del(lru_list[lru_list.index(lru_region)]) - self._memory_size -= lru_region.size() - self._handle_count -= 1 - #END while there is more memory to free - return num_found - - def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, - see MapCursor.use_region. - :param a: A regions (a)rray - :return: The newly created region""" - if self._memory_size + size > self._max_memory_size: - self._collect_lru_region(size) - #END handle collection - - r = None - if a: - assert len(a) == 1 - r = a[0] - else: - try: - r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions - - self._handle_count += 1 - self._memory_size += r.size() - a.append(r) - # END handle array - - assert r.includes_ofs(offset) - return r + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. + + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. + + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + acomodate this fact""" + + __slots__ = [ + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + #{ Configuration + MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion + WindowCursorCls = WindowCursor + #} END configuration + + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + """initialize the manager with the given parameters. + :param window_size: if -1, a default window size will be chosen depending on + the operating system's architechture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default iwll be set dependning on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate + :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, + the manager will free as many handles as posisble""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size < 0: + coeff = 32 + if is_64_bit(): + coeff = 1024 + #END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 512 + if is_64_bit(): + coeff = 8192 + #END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + #END handle max memory size + + #{ Internal Methods + + def _collect_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :return: Amount of freed regions + + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): + lru_region = region + lru_list = regions + # END update lru_region + #END for each region + #END for each regions list + + if lru_region is None: + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + #END while there is more memory to free + return num_found + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) + #END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.append(r) + # END handle array + + assert r.includes_ofs(offset) + return r - #}END internal methods - - #{ Interface - def make_cursor(self, path_or_fd): - """ - :return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory - - **Note:** if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - - **Note:** file descriptors are problematic as they are not necessarily unique, as two - different files opened and closed in succession might have the same file descriptor id. - - **Note:** Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" - regions = self._fdict.get(path_or_fd) - if regions is None: - regions = self.MapRegionListCls(path_or_fd) - self._fdict[path_or_fd] = regions - # END obtain region for path - return self.WindowCursorCls(self, regions) - - def collect(self): - """Collect all available free-to-collect mapped regions - :return: Amount of freed handles""" - return self._collect_lru_region(0) - - def num_file_handles(self): - """:return: amount of file handles in use. Each mapped region uses one file handle""" - return self._handle_count - - def num_open_files(self): - """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) - - def window_size(self): - """:return: size of each window when allocating new regions""" - return self._window_size - - def mapped_memory_size(self): - """:return: amount of bytes currently mapped in total""" - return self._memory_size - - def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" - return self._max_handle_count - - def max_mapped_memory_size(self): - """:return: maximum amount of memory we may allocate""" - return self._max_memory_size - - #} END interface - - #{ Special Purpose Interface - - def force_map_handle_removal_win(self, base_path): - """ONLY AVAILABLE ON WINDOWS - On windows removing files is not allowed if anybody still has it opened. - If this process is ourselves, and if the whole process uses this memory - manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to - allow the respective operation after all. - The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep - the cursors alive will not be using it anymore. They need to be recreated ! - :return: Amount of closed handles - - **Note:** does nothing on non-windows platforms""" - if sys.platform != 'win32': - return - #END early bailout - - num_closed = 0 - for path, rlist in self._fdict.iteritems(): - if path.startswith(base_path): - for region in rlist: - region._mf.close() - num_closed += 1 - #END path matches - #END for each path - return num_closed - #} END special purpose interface - - - + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.WindowCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + + **Note:** does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface + + + class SlidingWindowMapManager(StaticWindowMapManager): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily - obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles - which result from each mmap call, the least recently used, and currently unused mapped regions - are unloaded automatically. - - **Note:** currently not thread-safe ! - - **Note:** in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than - a safe amount of memory already, which would possibly cause memory allocations to fail as our address - space is full.""" - - __slots__ = tuple() - - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): - """Adjusts the default window size to -1""" - super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - - def _obtain_region(self, a, offset, size, flags, is_recursive): - # bisect to find an existing region. The c++ implementation cannot - # do that as it uses a linked list for regions. - r = None - lo = 0 - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - ofs = a[mid]._b - if ofs <= offset: - if a[mid].includes_ofs(offset): - r = a[mid] - break - #END have region - lo = mid+1 - else: - hi = mid - #END handle position - #END while bisecting - - if r is None: - window_size = self._window_size - left = self.MapWindowCls(0, 0) - mid = self.MapWindowCls(offset, size) - right = self.MapWindowCls(a.file_size(), 0) - - # we want to honor the max memory size, and assure we have anough - # memory available - # Save calls ! - if self._memory_size + window_size > self._max_memory_size: - self._collect_lru_region(window_size) - #END handle collection - - # we assume the list remains sorted by offset - insert_pos = 0 - len_regions = len(a) - if len_regions == 1: - if a[0]._b <= offset: - insert_pos = 1 - #END maintain sort - else: - # find insert position - insert_pos = len_regions - for i, region in enumerate(a): - if region._b > offset: - insert_pos = i - break - #END if insert position is correct - #END for each region - # END obtain insert pos - - # adjust the actual offset and size values to create the largest - # possible mapping - if insert_pos == 0: - if len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side - else: - if insert_pos != len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - # END adjust right window - left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows - - mid.extend_left_to(left, window_size) - mid.extend_right_to(right, window_size) - mid.align() - - # it can happen that we align beyond the end of the file - if mid.ofs_end() > right.ofs: - mid.size = right.ofs - mid.ofs - #END readjust size - - # insert new region at the right offset to keep the order - try: - if self._handle_count >= self._max_handle_count: - raise Exception - #END assert own imposed max file handles - r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions - - self._handle_count += 1 - self._memory_size += r.size() - a.insert(insert_pos, r) - # END create new region - return r - - + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to -1""" + super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + + def _obtain_region(self, a, offset, size, flags, is_recursive): + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + #END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r + + diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 6957dcab0..21e6c5a09 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -9,57 +9,57 @@ #{ Utilities class FileCreator(object): - """A instance which creates a temporary file with a prefix and a given size - and provides this info to the user. - Once it gets deleted, it will remove the temporary file as well.""" - __slots__ = ("_size", "_path") - - def __init__(self, size, prefix=''): - assert size, "Require size to be larger 0" - - self._path = tempfile.mktemp(prefix=prefix) - self._size = size - - fp = open(self._path, "wb") - fp.seek(size-1) - fp.write('1') - fp.close() - - assert os.path.getsize(self.path) == size + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._path = tempfile.mktemp(prefix=prefix) + self._size = size + + fp = open(self._path, "wb") + fp.seek(size-1) + fp.write('1') + fp.close() + + assert os.path.getsize(self.path) == size - def __del__(self): - try: - os.remove(self.path) - except OSError: - pass - #END exception handling - + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + #END exception handling + - @property - def path(self): - return self._path - - @property - def size(self): - return self._size + @property + def path(self): + return self._path + + @property + def size(self): + return self._size #} END utilities class TestBase(TestCase): - """Foundation used by all tests""" - - #{ Configuration - k_window_test_size = 1000 * 1000 * 8 + 5195 - #} END configuration - - #{ Overrides - @classmethod - def setUpAll(cls): - # nothing for now - pass - - #END overrides - - #{ Interface - - #} END interface + """Foundation used by all tests""" + + #{ Configuration + k_window_test_size = 1000 * 1000 * 8 + 5195 + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + #END overrides + + #{ Interface + + #} END interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 9881c6294..4bdcb76f5 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -10,108 +10,108 @@ man_optimal = SlidingWindowMapManager() -man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, - max_memory_size=TestBase.k_window_test_size/3, - max_open_handles=15) +man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, + max_memory_size=TestBase.k_window_test_size/3, + max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): - - def test_basics(self): - fc = FileCreator(self.k_window_test_size, "buffer_test") - - # invalid paths fail upon construction - c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - - buf = SlidingWindowMapBuffer() # can create uninitailized buffers - assert buf.cursor() is None - - # can call end access any time - buf.end_access() - buf.end_access() - assert len(buf) == 0 - - # begin access can revive it, if the offset is suitable - offset = 100 - assert buf.begin_access(c, fc.size) == False - assert buf.begin_access(c, offset) == True - assert len(buf) == fc.size - offset - assert buf.cursor().is_valid() - - # empty begin access keeps it valid on the same path, but alters the offset - assert buf.begin_access() == True - assert len(buf) == fc.size - assert buf.cursor().is_valid() - - # simple access - data = open(fc.path, 'rb').read() - assert data[offset] == buf[0] - assert data[offset:offset*2] == buf[0:offset] - - # negative indices, partial slices - assert buf[-1] == buf[len(buf)-1] - assert buf[-10:] == buf[len(buf)-10:len(buf)] - - # end access makes its cursor invalid - buf.end_access() - assert not buf.cursor().is_valid() - assert buf.cursor().is_associated() # but it remains associated - - # an empty begin access fixes it up again - assert buf.begin_access() == True and buf.cursor().is_valid() - del(buf) # ends access automatically - del(c) - - assert man_optimal.num_file_handles() == 1 - - # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to - # exagerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which - # will produce small mappings only ! - max_num_accesses = 100 - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case'), - (static_man, 'static optimial')): - buf = SlidingWindowMapBuffer(manager.make_cursor(item)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - #END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000*1000) - mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) - # END handle access mode - # END for each manager - # END for each input - os.close(fd) + + def test_basics(self): + fc = FileCreator(self.k_window_test_size, "buffer_test") + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitailized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + data = open(fc.path, 'rb').read() + assert data[offset] == buf[0] + assert data[offset:offset*2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf)-1] + assert buf[-10:] == buf[len(buf)-10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with rnadom access and a full mapping - we don't want to + # exagerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimial')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 27be686ad..46429a419 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -12,203 +12,203 @@ from copy import copy class TestMMan(TestBase): - - def test_cursor(self): - fc = FileCreator(self.k_window_test_size, "cursor_test") - - man = SlidingWindowMapManager() - ci = WindowCursor(man) # invalid cursor - assert not ci.is_valid() - assert not ci.is_associated() - assert ci.size() == 0 # this is cached, so we can query it in invalid state - - cv = man.make_cursor(fc.path) - assert not cv.is_valid() # no region mapped yet - assert cv.is_associated()# but it know where to map it from - assert cv.file_size() == fc.size - assert cv.path() == fc.path - - # copy module - cio = copy(cv) - assert not cio.is_valid() and cio.is_associated() - - # assign method - assert not ci.is_associated() - ci.assign(cv) - assert not ci.is_valid() and ci.is_associated() - - # unuse non-existing region is fine - cv.unuse_region() - cv.unuse_region() - - # destruction is fine (even multiple times) - cv._destroy() - WindowCursor(man)._destroy() - - def test_memory_manager(self): - slide_man = SlidingWindowMapManager() - static_man = StaticWindowMapManager() - - for man in (static_man, slide_man): - assert man.num_file_handles() == 0 - assert man.num_open_files() == 0 - winsize_cmp_val = 0 - if isinstance(man, StaticWindowMapManager): - winsize_cmp_val = -1 - #END handle window size - assert man.window_size() > winsize_cmp_val - assert man.mapped_memory_size() == 0 - assert man.max_mapped_memory_size() > 0 - - # collection doesn't raise in 'any' mode - man._collect_lru_region(0) - # doesn't raise if we are within the limit - man._collect_lru_region(10) - - # doesn't fail if we overallocate - assert man._collect_lru_region(sys.maxint) == 0 - - # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] - - if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) - else: - self.failUnlessRaises(ValueError, c.fd) - #END handle value error - #END for each input - os.close(fd) - # END for each manager type - - def test_memman_operation(self): - # test more access, force it to actually unmap regions - fc = FileCreator(self.k_window_test_size, "manager_operation_test") - data = open(fc.path, 'rb').read() - fd = os.open(fc.path, os.O_RDONLY) - max_num_handles = 15 - #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - - #assert c.size() == size # the cursor may overallocate in its static version - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - if man.window_size(): - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) - else: - assert rr().size() == fc.size - #END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - - # obtain second window, which spans the first part of the file - it is a still the same window - nsize = (size or fc.size) - 10 - assert c.use_region(0, nsize).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == nsize - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:nsize] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - (size or c.size()) + overshoot - assert c.use_region(base_offset, size).is_valid() - if man.window_size(): - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - else: - assert c.size() < fc.size - #END ignore static managers which only have one handle per file - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - #END ignore this for static managers - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) - - # precondition - if man.window_size(): - assert max_mapped_memory_size >= mapped_memory_size() - #END statics will overshoot, which is fine - assert max_file_handles >= num_file_handles() - assert c.use_region(base_offset, (size or c.size())).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize - - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - #END for each item - # END for each manager type - os.close(fd) + + def test_cursor(self): + fc = FileCreator(self.k_window_test_size, "cursor_test") + + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated()# but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path + + # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() + + # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + + # destruction is fine (even multiple times) + cv._destroy() + WindowCursor(man)._destroy() + + def test_memory_manager(self): + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() + + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + #END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + + # doesn't fail if we overallocate + assert man._collect_lru_region(sys.maxint) == 0 + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error + #END for each input + os.close(fd) + # END for each manager type + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + fc = FileCreator(self.k_window_test_size, "manager_operation_test") + data = open(fc.path, 'rb').read() + fd = os.open(fc.path, os.O_RDONLY) + max_num_handles = 15 + #small_size = + for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() / 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + + #assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + else: + assert rr().size() == fc.size + #END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + #END ignore static managers which only have one handle per file + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + #END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + #END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + # END for each manager type + os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index a9f4b1c08..4e1a5764b 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,83 +1,83 @@ from lib import TestBase class TestTutorial(TestBase): - - def test_example(self): - # Memory Managers - ################## - import smmap - # This instance should be globally available in your application - # It is configured to be well suitable for 32-bit or 64 bit applications. - mman = smmap.SlidingWindowMapManager() - - # the manager provides much useful information about its current state - # like the amount of open file handles or the amount of mapped memory - assert mman.num_file_handles() == 0 - assert mman.mapped_memory_size() == 0 - # and many more ... - - # Cursors - ########## - import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") - - # obtain a cursor to access some file. - c = mman.make_cursor(fc.path) - - # the cursor is now associated with the file, but not yet usable - assert c.is_associated() - assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to - # access. The following just says you want as much data as possible starting - # from offset 0. - # To be sure your region could be mapped, query for validity - assert c.use_region().is_valid() # use_region returns self - - # once a region was mapped, you must query its dimension regularly - # to assure you don't try to access its buffer out of its bounds - assert c.size() - c.buffer()[0] # first byte - c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size()-1] # last byte - - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - - # you can query absolute offsets, and check whether an offset is included - # in the cursor's data. - assert c.ofs_begin() < c.ofs_end() - assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the - # cursor will be come invalid. It cannot be used in that state - assert not c.use_region(fc.size, 100).is_valid() - # map as much as possible after skipping the first 100 bytes - assert c.use_region(100).is_valid() - - # You can explicitly free cursor resources by unusing the cursor's region - c.unuse_region() - assert not c.is_valid() - - # Buffers - ######### - # Create a default buffer which can operate on the whole file - buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - - # you can use it right away - assert buf.cursor().is_valid() - - buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:]# access the last ten bytes - - # If you want to keep the instance between different accesses, use the - # dedicated methods - buf.end_access() - assert not buf.cursor().is_valid() # you cannot use the buffer anymore - assert buf.begin_access(offset=10) # start using the buffer at an offset - - # it will stop using resources automatically once it goes out of scope - + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 096c5f6df..2df0660be 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -6,109 +6,109 @@ import sys class TestMMan(TestBase): - - def test_window(self): - wl = MapWindow(0, 1) # left - wc = MapWindow(1, 1) # center - wc2 = MapWindow(10, 5) # another center - wr = MapWindow(8000, 50) # right - - assert wl.ofs_end() == 1 - assert wc.ofs_end() == 2 - assert wr.ofs_end() == 8050 - - # extension does nothing if already in place - maxsize = 100 - wc.extend_left_to(wl, maxsize) - assert wc.ofs == 1 and wc.size == 1 - wl.extend_right_to(wc, maxsize) - wl.extend_right_to(wc, maxsize) - assert wl.ofs == 0 and wl.size == 1 - - # an actual left extension - pofs_end = wc2.ofs_end() - wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - - # respects maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - - # without maxsize - wc.extend_right_to(wr, sys.maxint) - assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - - # extend left - wr.extend_left_to(wc2, maxsize) - wr.extend_left_to(wc2, maxsize) - assert wr.size == maxsize - - wr.extend_left_to(wc2, sys.maxint) - assert wr.ofs == wc2.ofs_end() - - wc.align() - assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) - - def test_region(self): - fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size / 2 - rofs = align_to_mmap(4200, False) - rfull = MapRegion(fc.path, 0, fc.size) - rhalfofs = MapRegion(fc.path, rofs, fc.size) - rhalfsize = MapRegion(fc.path, 0, half_size) - - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always - - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - # with the values we have, this test only works on windows where an alignment - # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions - # as they use different mapping techniques to circumvent the missing offset - # argument of mmap. - if sys.platform != 'win32': - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - #END handle platforms - - # auto-refcount - assert rfull.client_count() == 1 - rfull2 = rfull - assert rfull.client_count() == 2 - - # usage - assert rfull.usage_count() == 0 - rfull.increment_usage_count() - assert rfull.usage_count() == 1 - - # window constructor - w = MapWindow.from_region(rfull) - assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - - def test_region_list(self): - fc = FileCreator(100, "sample_file") - - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - ml = MapRegionList(item) - - assert ml.client_count() == 1 - - assert len(ml) == 0 - assert ml.path_or_fd() == item - assert ml.file_size() == fc.size - #END handle input - os.close(fd) - - def test_util(self): - assert isinstance(is_64_bit(), bool) # just call it - assert align_to_mmap(1, False) == 0 - assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY - + + def test_window(self): + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) + + def test_region(self): + fc = FileCreator(self.k_window_test_size, "window_test") + half_size = fc.size / 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + # with the values we have, this test only works on windows where an alignment + # size of 4096 is assumed. + # We only test on linux as it is inconsitent between the python versions + # as they use different mapping techniques to circumvent the missing offset + # argument of mmap. + if sys.platform != 'win32': + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + #END handle platforms + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + + # usage + assert rfull.usage_count() == 0 + rfull.increment_usage_count() + assert rfull.usage_count() == 1 + + # window constructor + w = MapWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + fc = FileCreator(100, "sample_file") + + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert ml.client_count() == 1 + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + #END handle input + os.close(fd) + + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY + diff --git a/smmap/util.py b/smmap/util.py index b0fd83b3f..c6710b3fe 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -5,36 +5,36 @@ from mmap import mmap, ACCESS_READ try: - from mmap import ALLOCATIONGRANULARITY + from mmap import ALLOCATIONGRANULARITY except ImportError: - # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly - # useful for aligning the offset. The offset argument doesn't exist there though - from mmap import PAGESIZE as ALLOCATIONGRANULARITY + # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly + # useful for aligning the offset. The offset argument doesn't exist there though + from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance from sys import getrefcount -__all__ = [ "align_to_mmap", "is_64_bit", - "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] +__all__ = [ "align_to_mmap", "is_64_bit", + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities def align_to_mmap(num, round_up): - """ - Align the given integer number to the closest page offset, which usually is 4096 bytes. - - :param round_up: if True, the next higher multiple of page size is used, otherwise - the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) - :return: num rounded to closest page""" - res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; - if round_up and (res != num): - res += ALLOCATIONGRANULARITY - #END handle size - return res; - + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + if round_up and (res != num): + res += ALLOCATIONGRANULARITY + #END handle size + return res; + def is_64_bit(): - """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxint > (1<<32) - 1 + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxint > (1<<32) - 1 #}END utilities @@ -42,228 +42,228 @@ def is_64_bit(): #{ Utility Classes class MapWindow(object): - """Utility type which is used to snap windows towards each other, and to adjust their size""" - __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes - ) + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) - def __init__(self, offset, size): - self.ofs = offset - self.size = size + def __init__(self, offset, size): + self.ofs = offset + self.size = size - def __repr__(self): - return "MapWindow(%i, %i)" % (self.ofs, self.size) + def __repr__(self): + return "MapWindow(%i, %i)" % (self.ofs, self.size) - @classmethod - def from_region(cls, region): - """:return: new window from a region""" - return cls(region._b, region.size()) + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region._b, region.size()) - def ofs_end(self): - return self.ofs + self.size + def ofs_end(self): + return self.ofs + self.size - def align(self): - """Assures the previous window area is contained in the new one""" - nofs = align_to_mmap(self.ofs, 0) - self.size += self.ofs - nofs # keep size constant - self.ofs = nofs - self.size = align_to_mmap(self.size, 1) + def align(self): + """Assures the previous window area is contained in the new one""" + nofs = align_to_mmap(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs + self.size = align_to_mmap(self.size, 1) - def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, - but don't make yourself larger than max_size. - The resize will assure that the new window still contains the old window area""" - rofs = self.ofs - window.ofs_end() - nsize = rofs + self.size - rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs - self.size += rofs + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs - def extend_right_to(self, window, max_size): - """Adjust the size to make our window end where the right window begins, but don't - get larger than max_size""" - self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) class MapRegion(object): - """Defines a mapped region of memory, aligned to pagesizes - - **Note:** deallocates used region automatically on destruction""" - __slots__ = [ - '_b' , # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_size', # cached size of our memory map - '__weakref__' - ] - _need_compat_layer = sys.version_info[1] < 6 - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot - - #{ Configuration - # Used for testing only. If True, all data will be loaded into memory at once. - # This makes sure no file handles will remain open. - _test_read_into_memory = False - #} END configuration - - - def __init__(self, path_or_fd, ofs, size, flags = 0): - """Initialize a region, allocate the memory map - :param path_or_fd: path to the file to map, or the opened file descriptor - :param ofs: **aligned** offset into the file to be mapped - :param size: if size is larger then the file on disk, the whole file will be - allocated the the size automatically adjusted - :param flags: additional flags to be given when opening the file. - :raise Exception: if no memory can be allocated""" - self._b = ofs - self._size = 0 - self._uc = 0 - - if isinstance(path_or_fd, int): - fd = path_or_fd - else: - fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) - #END handle fd - - try: - kwargs = dict(access=ACCESS_READ, offset=ofs) - corrected_size = size - sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will - # bark that the size is too large ... many extra file accesses because - # if this ... argh ! - actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) - if self._test_read_into_memory: - self._mf = self._read_into_memory(fd, ofs, actual_size) - else: - self._mf = mmap(fd, actual_size, **kwargs) - #END handle memory mode - - self._size = len(self._mf) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, self._size) - #END handle buffer wrapping - finally: - if isinstance(path_or_fd, basestring): - os.close(fd) - #END only close it if we opened it - #END close file handle - - def _read_into_memory(self, fd, offset, size): - """:return: string data as read from the given file descriptor, offset and size """ - os.lseek(fd, offset, os.SEEK_SET) - mf = '' - bytes_todo = size - while bytes_todo: - chunk = 1024*1024 - d = os.read(fd, chunk) - bytes_todo -= len(d) - mf += d - #END loop copy items - return mf - - def __repr__(self): - return "MapRegion<%i, %i>" % (self._b, self.size()) - - #{ Interface + """Defines a mapped region of memory, aligned to pagesizes + + **Note:** deallocates used region automatically on destruction""" + __slots__ = [ + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] + _need_compat_layer = sys.version_info[1] < 6 + + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + + #{ Configuration + # Used for testing only. If True, all data will be loaded into memory at once. + # This makes sure no file handles will remain open. + _test_read_into_memory = False + #} END configuration + + + def __init__(self, path_or_fd, ofs, size, flags = 0): + """Initialize a region, allocate the memory map + :param path_or_fd: path to the file to map, or the opened file descriptor + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._size = 0 + self._uc = 0 + + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + #END handle fd + + try: + kwargs = dict(access=ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + sizeofs = 0 + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + if self._test_read_into_memory: + self._mf = self._read_into_memory(fd, ofs, actual_size) + else: + self._mf = mmap(fd, actual_size, **kwargs) + #END handle memory mode + + self._size = len(self._mf) + + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, self._size) + #END handle buffer wrapping + finally: + if isinstance(path_or_fd, basestring): + os.close(fd) + #END only close it if we opened it + #END close file handle + + def _read_into_memory(self, fd, offset, size): + """:return: string data as read from the given file descriptor, offset and size """ + os.lseek(fd, offset, os.SEEK_SET) + mf = '' + bytes_todo = size + while bytes_todo: + chunk = 1024*1024 + d = os.read(fd, chunk) + bytes_todo -= len(d) + mf += d + #END loop copy items + return mf + + def __repr__(self): + return "MapRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface - def buffer(self): - """:return: a buffer containing the memory""" - return self._mf - - def map(self): - """:return: a memory map containing the memory""" - return self._mf - - def ofs_begin(self): - """:return: absolute byte offset to the first byte of the mapping""" - return self._b - - def size(self): - """:return: total size of the mapped region in bytes""" - return self._size - - def ofs_end(self): - """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self._size - - def includes_ofs(self, ofs): - """:return: True if the given offset can be read in our mapped region""" - return self._b <= ofs < self._b + self._size - - def client_count(self): - """:return: number of clients currently using this region""" - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 - - def usage_count(self): - """:return: amount of usages so far""" - return self._uc - - def increment_usage_count(self): - """Adjust the usage count by the given positive or negative offset""" - self._uc += 1 - - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return self._size - self._b - - def ofs_end(self): - # always the size - we are as large as it gets - return self._size - - def buffer(self): - return self._mfb - - def includes_ofs(self, ofs): - return self._b <= ofs < self._size - #END handle compat layer - - #} END interface - - + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + + def map(self): + """:return: a memory map containing the memory""" + return self._mf + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return self._size + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self._size + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return self._b <= ofs < self._b + self._size + + def client_count(self): + """:return: number of clients currently using this region""" + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def increment_usage_count(self): + """Adjust the usage count by the given positive or negative offset""" + self._uc += 1 + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return self._size - self._b + + def ofs_end(self): + # always the size - we are as large as it gets + return self._size + + def buffer(self): + return self._mfb + + def includes_ofs(self, ofs): + return self._b <= ofs < self._size + #END handle compat layer + + #} END interface + + class MapRegionList(list): - """List of MapRegion instances associating a path with a list of regions.""" - __slots__ = ( - '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map - ) - - def __new__(cls, path): - return super(MapRegionList, cls).__new__(cls) - - def __init__(self, path_or_fd): - self._path_or_fd = path_or_fd - self._file_size = None - - def client_count(self): - """:return: amount of clients which hold a reference to this instance""" - return getrefcount(self)-3 - - def path_or_fd(self): - """:return: path or file descriptor we are attached to""" - return self._path_or_fd - - def file_size(self): - """:return: size of file we manager""" - if self._file_size is None: - if isinstance(self._path_or_fd, basestring): - self._file_size = os.stat(self._path_or_fd).st_size - else: - self._file_size = os.fstat(self._path_or_fd).st_size - #END handle path type - #END update file size - return self._file_size - + """List of MapRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MapRegionList, cls).__new__(cls) + + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd + self._file_size = None + + def client_count(self): + """:return: amount of clients which hold a reference to this instance""" + return getrefcount(self)-3 + + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + if isinstance(self._path_or_fd, basestring): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + #END handle path type + #END update file size + return self._file_size + #} END utilty classes From 6576d5503a64d124fd7bcf639cc8955918b3ac43 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Feb 2014 20:51:43 +0100 Subject: [PATCH 0204/1790] tabs to spaces --- gitdb/__init__.py | 22 +- gitdb/base.py | 556 ++--- gitdb/db/base.py | 586 ++--- gitdb/db/git.py | 134 +- gitdb/db/loose.py | 470 ++-- gitdb/db/mem.py | 188 +- gitdb/db/pack.py | 374 ++-- gitdb/db/ref.py | 138 +- gitdb/exc.py | 30 +- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 1206 +++++------ gitdb/pack.py | 1920 ++++++++--------- gitdb/stream.py | 1288 +++++------ gitdb/test/__init__.py | 8 +- gitdb/test/db/lib.py | 376 ++-- gitdb/test/db/test_git.py | 74 +- gitdb/test/db/test_loose.py | 50 +- gitdb/test/db/test_mem.py | 46 +- gitdb/test/db/test_pack.py | 118 +- gitdb/test/db/test_ref.py | 98 +- gitdb/test/lib.py | 206 +- gitdb/test/performance/lib.py | 56 +- gitdb/test/performance/test_pack.py | 150 +- gitdb/test/performance/test_pack_streaming.py | 120 +- gitdb/test/performance/test_stream.py | 324 +-- gitdb/test/test_base.py | 168 +- gitdb/test/test_example.py | 102 +- gitdb/test/test_pack.py | 438 ++-- gitdb/test/test_stream.py | 266 +-- gitdb/test/test_util.py | 184 +- gitdb/util.py | 538 ++--- 32 files changed, 5119 insertions(+), 5119 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 800b292da..ff750d14c 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -9,17 +9,17 @@ #{ Initialization def _init_externals(): - """Initialize external projects by putting them into the path""" - for module in ('async', 'smmap'): - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - - try: - __import__(module) - except ImportError: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) - #END verify import - #END handel imports - + """Initialize external projects by putting them into the path""" + for module in ('async', 'smmap'): + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + + try: + __import__(module) + except ImportError: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + #END verify import + #END handel imports + #} END initialization _init_externals() diff --git a/gitdb/base.py b/gitdb/base.py index ff1062bf6..bad5f7472 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -4,308 +4,308 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( - bin_to_hex, - zlib - ) + bin_to_hex, + zlib + ) from fun import ( - type_id_to_type_map, - type_to_type_id_map - ) + type_id_to_type_map, + type_to_type_id_map + ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream' ) #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provding information - about the binary sha of the object, the type_string as well as the uncompressed size - in bytes. - - It can be accessed using tuple notation and using attribute access notation:: - - assert dbi[0] == dbi.binsha - assert dbi[1] == dbi.type - assert dbi[2] == dbi.size - - The type is designed to be as lighteight as possible.""" - __slots__ = tuple() - - def __new__(cls, sha, type, size): - return tuple.__new__(cls, (sha, type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return self[0] - - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return bin_to_hex(self[0]) - - @property - def type(self): - return self[1] - - @property - def type_id(self): - return type_to_type_id_map[self[1]] - - @property - def size(self): - return self[2] - #} END interface - - + """Carries information about an object in an ODB, provding information + about the binary sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.binsha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return self[0] + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + @property + def type(self): + return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] + + @property + def size(self): + return self[2] + #} END interface + + class OPackInfo(tuple): - """As OInfo, but provides a type_id property to retrieve the numerical type id, and - does not include a sha. - - Additionally, the pack_offset is the absolute offset into the packfile at which - all object information is located. The data_offset property points to the abosolute - location in the pack at which that actual data stream can be found.""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size): - return tuple.__new__(cls, (packoffset,type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - - @property - def pack_offset(self): - return self[0] - - @property - def type(self): - return type_id_to_type_map[self[1]] - - @property - def type_id(self): - return self[1] - - @property - def size(self): - return self[2] - - #} END interface - - + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the abosolute + location in the pack at which that actual data stream can be found.""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset,type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def pack_offset(self): + return self[0] + + @property + def type(self): + return type_id_to_type_map[self[1]] + + @property + def type_id(self): + return self[1] + + @property + def size(self): + return self[2] + + #} END interface + + class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, - or the negative offset from the pack_offset, so that pack_offset - delta_info yields - the pack offset of the base object""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, delta_info): - return tuple.__new__(cls, (packoffset, type, size, delta_info)) - - #{ Interface - @property - def delta_info(self): - return self[3] - #} END interface - - + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional - information about the stream. - Generally, ODB streams are read-only as objects are immutable""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - - def __init__(self, *args, **kwargs): - tuple.__init__(self) - - #{ Stream Reader Interface - - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - - #} END stream reader interface - - + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + + #} END stream reader interface + + class ODeltaStream(OStream): - """Uses size info of its stream, delaying reads""" - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - #{ Stream Reader Interface - - @property - def size(self): - return self[3].size - - #} END stream reader interface - - + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + + #} END stream reader interface + + class OPackStream(OPackInfo): - """Next to pack object information, a stream outputting an undeltified base object - is provided""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, stream, *args): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (packoffset, type, size, stream)) - - #{ Stream Reader Interface - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (packoffset, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface - + class ODeltaPackStream(ODeltaPackInfo): - """Provides a stream outputting the uncompressed offset delta information""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, delta_info, stream): - return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface - def read(self, size=-1): - return self[4].read(size) - - @property - def stream(self): - return self[4] - #} END stream reader interface + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] + #} END stream reader interface class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow - the ODB to record information about the operations outcome right in this instance. - - It provides interfaces for the OStream and a StreamReader to allow the instance - to blend in without prior conversion. - - The only method your content stream must support is 'read'""" - __slots__ = tuple() - - def __new__(cls, type, size, stream, sha=None): - return list.__new__(cls, (sha, type, size, stream, None)) - - def __init__(self, type, size, stream, sha=None): - list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return bin_to_hex(self[0]) - - def _error(self): - """:return: the error that occurred when processing the stream, or None""" - return self[4] - - def _set_error(self, exc): - """Set this input stream to the given exc, may be None to reset the error""" - self[4] = exc - - error = property(_error, _set_error) - - #} END interface - - #{ Stream Reader Interface - - def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on - to our internal stream""" - return self[3].read(size) - - #} END stream reader interface - - #{ interface - - def _set_binsha(self, binsha): - self[0] = binsha - - def _binsha(self): - return self[0] - - binsha = property(_binsha, _set_binsha) - - - def _type(self): - return self[1] - - def _set_type(self, type): - self[1] = type - - type = property(_type, _set_type) - - def _size(self): - return self[2] - - def _set_size(self, size): - self[2] = size - - size = property(_size, _set_size) - - def _stream(self): - return self[3] - - def _set_stream(self, stream): - self[3] = stream - - stream = property(_stream, _set_stream) - - #} END odb info interface - + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_binsha(self, binsha): + self[0] = binsha + + def _binsha(self): + return self[0] + + binsha = property(_binsha, _set_binsha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in - the queried database. The exception attribute provides more information about - the cause of the issue""" - __slots__ = tuple() - - def __new__(cls, sha, exc): - return tuple.__new__(cls, (sha, exc)) - - def __init__(self, sha, exc): - tuple.__init__(self, (sha, exc)) - - @property - def binsha(self): - return self[0] - - @property - def hexsha(self): - return bin_to_hex(self[0]) - - @property - def error(self): - """:return: exception instance explaining the failure""" - return self[1] + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def binsha(self): + return self[0] + + @property + def hexsha(self): + return bin_to_hex(self[0]) + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] class InvalidOStream(InvalidOInfo): - """Carries information about an invalid ODB stream""" - __slots__ = tuple() - + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + #} END ODB Bases diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 984acafbf..867e93a81 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,20 +4,20 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, - join, - LazyMixin, - hex_to_bin - ) + pool, + join, + LazyMixin, + hex_to_bin + ) from gitdb.exc import ( - BadObject, - AmbiguousObjectName - ) + BadObject, + AmbiguousObjectName + ) from async import ( - ChannelThreadTask - ) + ChannelThreadTask + ) from itertools import chain @@ -26,301 +26,301 @@ class ObjectDBR(object): - """Defines an interface for object database lookup. - Objects are identified either by their 20 byte bin sha""" - - def __contains__(self, sha): - return self.has_obj - - #{ Query Interface - def has_object(self, sha): - """ - :return: True if the object identified by the given 20 bytes - binary sha is contained in the database""" - raise NotImplementedError("To be implemented in subclass") - - def has_object_async(self, reader): - """Return a reader yielding information about the membership of objects - as identified by shas - :param reader: Reader yielding 20 byte shas. - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate - whether the given sha exists in the database or not""" - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - - def info(self, sha): - """ :return: OInfo instance - :param sha: bytes binary sha - :raise BadObject:""" - raise NotImplementedError("To be implemented in subclass") - - def info_async(self, reader): - """Retrieve information of a multitude of objects asynchronously - :param reader: Channel yielding the sha's of the objects of interest - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" - task = ChannelThreadTask(reader, str(self.info_async), self.info) - return pool.add_task(task) - - def stream(self, sha): - """:return: OStream instance - :param sha: 20 bytes binary sha - :raise BadObject:""" - raise NotImplementedError("To be implemented in subclass") - - def stream_async(self, reader): - """Retrieve the OStream of multiple objects - :param reader: see ``info`` - :param max_threads: see ``ObjectDBW.store`` - :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to - read all OStreams at once. Instead, read them individually using reader.read(x) - where x is small enough.""" - # base implementation just uses the stream method repeatedly - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) - return pool.add_task(task) - - def size(self): - """:return: amount of objects in this database""" - raise NotImplementedError() - - def sha_iter(self): - """Return iterator yielding 20 byte shas for all objects in this data base""" - raise NotImplementedError() - - #} END query interface - - + """Defines an interface for object database lookup. + Objects are identified either by their 20 byte bin sha""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + :return: True if the object identified by the given 20 bytes + binary sha is contained in the database""" + raise NotImplementedError("To be implemented in subclass") + + def has_object_async(self, reader): + """Return a reader yielding information about the membership of objects + as identified by shas + :param reader: Reader yielding 20 byte shas. + :return: async.Reader yielding tuples of (sha, bool) pairs which indicate + whether the given sha exists in the database or not""" + task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) + return pool.add_task(task) + + def info(self, sha): + """ :return: OInfo instance + :param sha: bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info_async(self, reader): + """Retrieve information of a multitude of objects asynchronously + :param reader: Channel yielding the sha's of the objects of interest + :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" + task = ChannelThreadTask(reader, str(self.info_async), self.info) + return pool.add_task(task) + + def stream(self, sha): + """:return: OStream instance + :param sha: 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream_async(self, reader): + """Retrieve the OStream of multiple objects + :param reader: see ``info`` + :param max_threads: see ``ObjectDBW.store`` + :return: async.Reader yielding OStream|InvalidOStream instances in any order + + **Note:** depending on the system configuration, it might not be possible to + read all OStreams at once. Instead, read them individually using reader.read(x) + where x is small enough.""" + # base implementation just uses the stream method repeatedly + task = ChannelThreadTask(reader, str(self.stream_async), self.stream) + return pool.add_task(task) + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() + + #} END query interface + + class ObjectDBW(object): - """Defines an interface to create objects in the database""" - - def __init__(self, *args, **kwargs): - self._ostream = None - - #{ Edit Interface - def set_ostream(self, stream): - """ - Adjusts the stream to which all data should be sent when storing new objects - - :param stream: if not None, the stream to use, if None the default stream - will be used. - :return: previously installed stream, or None if there was no override - :raise TypeError: if the stream doesn't have the supported functionality""" - cstream = self._ostream - self._ostream = stream - return cstream - - def ostream(self): - """ - :return: overridden output stream this instance will write to, or None - if it will write to the default stream""" - return self._ostream - - def store(self, istream): - """ - Create a new object in the database - :return: the input istream object with its sha set to its corresponding value - - :param istream: IStream compatible instance. If its sha is already set - to a value, the object will just be stored in the our database format, - in which case the input stream is expected to be in object format ( header + contents ). - :raise IOError: if data could not be written""" - raise NotImplementedError("To be implemented in subclass") - - def store_async(self, reader): - """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as - they are computed. - - :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, - or its error attribute will be set to the exception informing about the error. - - :param reader: async.Reader yielding IStream instances. - The same instances will be used in the output channel as were received - in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how - many items have actually been produced.""" - # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) - return pool.add_task(task) - - #} END edit interface - + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """ + Adjusts the stream to which all data should be sent when storing new objects + + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """ + :return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """ + Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + def store_async(self, reader): + """ + Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as + they are computed. + + :return: Channel yielding your IStream which served as input, in any order. + The IStreams sha will be set to the sha it received during the process, + or its error attribute will be set to the exception informing about the error. + + :param reader: async.Reader yielding IStream instances. + The same instances will be used in the output channel as were received + in by the Reader. + + **Note:** As some ODB implementations implement this operation atomic, they might + abort the whole operation if one item could not be processed. Hence check how + many items have actually been produced.""" + # base implementation uses store to perform the work + task = ChannelThreadTask(reader, str(self.store_async), self.store) + return pool.add_task(task) + + #} END edit interface + class FileDBBase(object): - """Provides basic facilities to retrieve files of interest, including - caching facilities to help mapping hexsha's to objects""" - - def __init__(self, root_path): - """Initialize this instance to look for its files at the given root path - All subsequent operations will be relative to this path - :raise InvalidDBRoot: - **Note:** The base will not perform any accessablity checking as the base - might not yet be accessible, but become accessible before the first - access.""" - super(FileDBBase, self).__init__() - self._root_path = root_path - - - #{ Interface - def root_path(self): - """:return: path at which this db operates""" - return self._root_path - - def db_path(self, rela_path): - """ - :return: the given relative path relative to our database root, allowing - to pontentially access datafiles""" - return join(self._root_path, rela_path) - #} END interface - + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + **Note:** The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super(FileDBBase, self).__init__() + self._root_path = root_path + + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, rela_path) + #} END interface + class CachingDB(object): - """A database which uses caches to speed-up access""" - - #{ Interface - def update_cache(self, force=False): - """ - Call this method if the underlying data changed to trigger an update - of the internal caching structures. - - :param force: if True, the update must be performed. Otherwise the implementation - may decide not to perform an update if it thinks nothing has changed. - :return: True if an update was performed as something change indeed""" - - # END interface + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """ + Call this method if the underlying data changed to trigger an update + of the internal caching structures. + + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed - and compound databases.""" - if isinstance(database, CompoundDB): - compounds = list() - dbs = database.databases() - output.extend(db for db in dbs if not isinstance(db, CompoundDB)) - for cdb in (db for db in dbs if isinstance(db, CompoundDB)): - _databases_recursive(cdb, output) - else: - output.append(database) - # END handle database type - + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): - """A database which delegates calls to sub-databases. - - Databases are stored in the lazy-loaded _dbs attribute. - Define _set_cache_ to update it with your databases""" - def _set_cache_(self, attr): - if attr == '_dbs': - self._dbs = list() - elif attr == '_db_cache': - self._db_cache = dict() - else: - super(CompoundDB, self)._set_cache_(attr) - - def _db_query(self, sha): - """:return: database containing the given 20 byte sha - :raise BadObject:""" - # most databases use binary representations, prevent converting - # it everytime a database is being queried - try: - return self._db_cache[sha] - except KeyError: - pass - # END first level cache - - for db in self._dbs: - if db.has_object(sha): - self._db_cache[sha] = db - return db - # END for each database - raise BadObject(sha) - - #{ ObjectDBR interface - - def has_object(self, sha): - try: - self._db_query(sha) - return True - except BadObject: - return False - # END handle exceptions - - def info(self, sha): - return self._db_query(sha).info(sha) - - def stream(self, sha): - return self._db_query(sha).stream(sha) + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super(CompoundDB, self)._set_cache_(attr) + + def _db_query(self, sha): + """:return: database containing the given 20 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it everytime a database is being queried + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + + for db in self._dbs: + if db.has_object(sha): + self._db_cache[sha] = db + return db + # END for each database + raise BadObject(sha) + + #{ ObjectDBR interface + + def has_object(self, sha): + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions + + def info(self, sha): + return self._db_query(sha).info(sha) + + def stream(self, sha): + return self._db_query(sha).stream(sha) - def size(self): - """:return: total size of all contained databases""" - return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) - - def sha_iter(self): - return chain(*(db.sha_iter() for db in self._dbs)) - - #} END object DBR Interface - - #{ Interface - - def databases(self): - """:return: tuple of database instances we use for lookups""" - return tuple(self._dbs) + def size(self): + """:return: total size of all contained databases""" + return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) + + def sha_iter(self): + return chain(*(db.sha_iter() for db in self._dbs)) + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) - def update_cache(self, force=False): - # something might have changed, clear everything - self._db_cache.clear() - stat = False - for db in self._dbs: - if isinstance(db, CachingDB): - stat |= db.update_cache(force) - # END if is caching db - # END for each database to update - return stat - - def partial_to_complete_sha_hex(self, partial_hexsha): - """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha - :param partial_hexsha: hexsha with less than 40 byte - :raise AmbiguousObjectName: """ - databases = list() - _databases_recursive(self, databases) - - len_partial_hexsha = len(partial_hexsha) - if len_partial_hexsha % 2 != 0: - partial_binsha = hex_to_bin(partial_hexsha + "0") - else: - partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - - candidate = None - for db in databases: - full_bin_sha = None - try: - if hasattr(db, 'partial_to_complete_sha_hex'): - full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) - else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) - # END handle database type - except BadObject: - continue - # END ignore bad objects - if full_bin_sha: - if candidate and candidate != full_bin_sha: - raise AmbiguousObjectName(partial_hexsha) - candidate = full_bin_sha - # END handle candidate - # END for each db - if not candidate: - raise BadObject(partial_binsha) - return candidate - - #} END interface - + def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface + diff --git a/gitdb/db/git.py b/gitdb/db/git.py index b8fc46aa0..1d6ad0f26 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -3,10 +3,10 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - CompoundDB, - ObjectDBW, - FileDBBase - ) + CompoundDB, + ObjectDBW, + FileDBBase + ) from loose import LooseObjectDB from pack import PackedDB @@ -14,72 +14,72 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) import os __all__ = ('GitDB', ) class GitDB(FileDBBase, ObjectDBW, CompoundDB): - """A git-style object database, which contains all objects in the 'objects' - subdirectory""" - # Configuration - PackDBCls = PackedDB - LooseDBCls = LooseObjectDB - ReferenceDBCls = ReferenceDB - - # Directories - packs_dir = 'pack' - loose_dir = '' - alternates_dir = os.path.join('info', 'alternates') - - def __init__(self, root_path): - """Initialize ourselves on a git objects directory""" - super(GitDB, self).__init__(root_path) - - def _set_cache_(self, attr): - if attr == '_dbs' or attr == '_loose_db': - self._dbs = list() - loose_db = None - for subpath, dbcls in ((self.packs_dir, self.PackDBCls), - (self.loose_dir, self.LooseDBCls), - (self.alternates_dir, self.ReferenceDBCls)): - path = self.db_path(subpath) - if os.path.exists(path): - self._dbs.append(dbcls(path)) - if dbcls is self.LooseDBCls: - loose_db = self._dbs[-1] - # END remember loose db - # END check path exists - # END for each db type - - # should have at least one subdb - if not self._dbs: - raise InvalidDBRoot(self.root_path()) - # END handle error - - # we the first one should have the store method - assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" - - # finally set the value - self._loose_db = loose_db - else: - super(GitDB, self)._set_cache_(attr) - # END handle attrs - - #{ ObjectDBW interface - - def store(self, istream): - return self._loose_db.store(istream) - - def ostream(self): - return self._loose_db.ostream() - - def set_ostream(self, ostream): - return self._loose_db.set_ostream(ostream) - - #} END objectdbw interface - + """A git-style object database, which contains all objects in the 'objects' + subdirectory""" + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB + + # Directories + packs_dir = 'pack' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super(GitDB, self).__init__(root_path) + + def _set_cache_(self, attr): + if attr == '_dbs' or attr == '_loose_db': + self._dbs = list() + loose_db = None + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + else: + super(GitDB, self)._set_cache_(attr) + # END handle attrs + + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + + #} END objectdbw interface + diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 6cd1cefd5..dc0ea0e3b 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -3,53 +3,53 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - FileDBBase, - ObjectDBR, - ObjectDBW - ) + FileDBBase, + ObjectDBR, + ObjectDBW + ) from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) from gitdb.stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - FDStream, - Sha1Writer - ) + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer + ) from gitdb.base import ( - OStream, - OInfo - ) + OStream, + OInfo + ) from gitdb.util import ( - file_contents_ro_filepath, - ENOENT, - hex_to_bin, - bin_to_hex, - exists, - chmod, - isdir, - isfile, - remove, - mkdir, - rename, - dirname, - basename, - join - ) + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isdir, + isfile, + remove, + mkdir, + rename, + dirname, + basename, + join + ) from gitdb.fun import ( - chunk_size, - loose_object_header_info, - write_object, - stream_copy - ) + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) import tempfile import mmap @@ -61,202 +61,202 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): - """A database which operates on loose object files""" - - # CONFIGURATION - # chunks in which data will be copied between streams - stream_chunk_size = chunk_size - - # On windows we need to keep it writable, otherwise it cannot be removed - # either - new_objects_mode = 0444 - if os.name == 'nt': - new_objects_mode = 0644 - - - def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) - self._hexsha_to_file = dict() - # Additional Flags - might be set to 0 after the first failure - # Depending on the root, this might work for some mounts, for others not, which - # is why it is per instance - self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface - def object_path(self, hexsha): - """ - :return: path at which the object with the given hexsha would be stored, - relative to the database root""" - return join(hexsha[:2], hexsha[2:]) - - def readable_db_object_path(self, hexsha): - """ - :return: readable object path to the object identified by hexsha - :raise BadObject: If the object file does not exist""" - try: - return self._hexsha_to_file[hexsha] - except KeyError: - pass - # END ignore cache misses - - # try filesystem - path = self.db_path(self.object_path(hexsha)) - if exists(path): - self._hexsha_to_file[hexsha] = path - return path - # END handle cache - raise BadObject(hexsha) - - def partial_to_complete_sha_hex(self, partial_hexsha): - """:return: 20 byte binary sha1 string which matches the given name uniquely - :param name: hexadecimal partial name - :raise AmbiguousObjectName: - :raise BadObject: """ - candidate = None - for binsha in self.sha_iter(): - if bin_to_hex(binsha).startswith(partial_hexsha): - # it can't ever find the same object twice - if candidate is not None: - raise AmbiguousObjectName(partial_hexsha) - candidate = binsha - # END for each object - if candidate is None: - raise BadObject(partial_hexsha) - return candidate - - #} END interface - - def _map_loose_object(self, sha): - """ - :return: memory map of that file to allow random read access - :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(bin_to_hex(sha))) - try: - return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) - except OSError,e: - if e.errno != ENOENT: - # try again without noatime - try: - return file_contents_ro_filepath(db_path) - except OSError: - raise BadObject(sha) - # didn't work because of our flag, don't try it again - self._fd_open_flags = 0 - else: - raise BadObject(sha) - # END handle error - # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed - - def set_ostream(self, stream): - """:raise TypeError: if the stream does not support the Sha1Writer interface""" - if stream is not None and not isinstance(stream, Sha1Writer): - raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) - - def info(self, sha): - m = self._map_loose_object(sha) - try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) - finally: - m.close() - # END assure release of system resources - - def stream(self, sha): - m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) - return OStream(sha, type, size, stream) - - def has_object(self, sha): - try: - self.readable_db_object_path(bin_to_hex(sha)) - return True - except BadObject: - return False - # END check existance - - def store(self, istream): - """note: The sha we produce will be hex by nature""" - tmp_path = None - writer = self.ostream() - if writer is None: - # open a tmp file to write the data to - fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - - if istream.binsha is None: - writer = FDCompressedSha1Writer(fd) - else: - writer = FDStream(fd) - # END handle direct stream copies - # END handle custom writer - - try: - try: - if istream.binsha is not None: - # copy as much as possible, the actual uncompressed item size might - # be smaller than the compressed version - stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) - else: - # write object with header, we have to make a new one - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) - # END handle direct stream copies - finally: - if tmp_path: - writer.close() - # END assure target stream is closed - except: - if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - - hexsha = None - if istream.binsha: - hexsha = istream.hexsha - else: - hexsha = writer.sha(as_hex=True) - # END handle sha - - if tmp_path: - obj_path = self.db_path(self.object_path(hexsha)) - obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) - # END handle destination directory - # rename onto existing doesn't work on windows - if os.name == 'nt' and isfile(obj_path): - remove(obj_path) - # END handle win322 - rename(tmp_path, obj_path) - - # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rwrr - chmod(obj_path, self.new_objects_mode) - # END handle dry_run - - istream.binsha = hex_to_bin(hexsha) - return istream - - def sha_iter(self): - # find all files which look like an object, extract sha from there - for root, dirs, files in os.walk(self.root_path()): - root_base = basename(root) - if len(root_base) != 2: - continue - - for f in files: - if len(f) != 38: - continue - yield hex_to_bin(root_base + f) - # END for each file - # END for each walk iteration - - def size(self): - return len(tuple(self.sha_iter())) - + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = 0444 + if os.name == 'nt': + new_objects_mode = 0644 + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(partial_hexsha): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(bin_to_hex(sha))) + try: + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + return file_contents_ro_filepath(db_path) + except OSError: + raise BadObject(sha) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(sha) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(bin_to_hex(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + + if istream.binsha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies + # END handle custom writer + + try: + try: + if istream.binsha is not None: + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + + hexsha = None + if istream.binsha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha + + if tmp_path: + obj_path = self.db_path(self.object_path(hexsha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + # rename onto existing doesn't work on windows + if os.name == 'nt' and isfile(obj_path): + remove(obj_path) + # END handle win322 + rename(tmp_path, obj_path) + + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + # END handle dry_run + + istream.binsha = hex_to_bin(hexsha) + return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) + diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 5d76c83cc..b9b2b8995 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -5,109 +5,109 @@ """Contains the MemoryDatabase implementation""" from loose import LooseObjectDB from base import ( - ObjectDBR, - ObjectDBW - ) + ObjectDBR, + ObjectDBW + ) from gitdb.base import ( - OStream, - IStream, - ) + OStream, + IStream, + ) from gitdb.exc import ( - BadObject, - UnsupportedOperation - ) + BadObject, + UnsupportedOperation + ) from gitdb.stream import ( - ZippedStoreShaWriter, - DecompressMemMapReader, - ) + ZippedStoreShaWriter, + DecompressMemMapReader, + ) from cStringIO import StringIO __all__ = ("MemoryDB", ) class MemoryDB(ObjectDBR, ObjectDBW): - """A memory database stores everything to memory, providing fast IO and object - retrieval. It should be used to buffer results and obtain SHAs before writing - it to the actual physical storage, as it allows to query whether object already - exists in the target storage before introducing actual IO - - **Note:** memory is currently not threadsafe, hence the async methods cannot be used - for storing""" - - def __init__(self): - super(MemoryDB, self).__init__() - self._db = LooseObjectDB("path/doesnt/matter") - - # maps 20 byte shas to their OStream objects - self._cache = dict() - - def set_ostream(self, stream): - raise UnsupportedOperation("MemoryDB's always stream into memory") - - def store(self, istream): - zstream = ZippedStoreShaWriter() - self._db.set_ostream(zstream) - - istream = self._db.store(istream) - zstream.close() # close to flush - zstream.seek(0) - - # don't provide a size, the stream is written in object format, hence the - # header needs decompression - decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) - self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) - - return istream - - def store_async(self, reader): - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - - def has_object(self, sha): - return sha in self._cache + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO + + **Note:** memory is currently not threadsafe, hence the async methods cannot be used + for storing""" + + def __init__(self): + super(MemoryDB, self).__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) + + return istream + + def store_async(self, reader): + raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") + + def has_object(self, sha): + return sha in self._cache - def info(self, sha): - # we always return streams, which are infos as well - return self.stream(sha) - - def stream(self, sha): - try: - ostream = self._cache[sha] - # rewind stream for the next one to read - ostream.stream.seek(0) - return ostream - except KeyError: - raise BadObject(sha) - # END exception handling - - def size(self): - return len(self._cache) - - def sha_iter(self): - return self._cache.iterkeys() - - - #{ Interface - def stream_copy(self, sha_iter, odb): - """Copy the streams as identified by sha's yielded by sha_iter into the given odb - The streams will be copied directly - **Note:** the object will only be written if it did not exist in the target db - :return: amount of streams actually copied into odb. If smaller than the amount - of input shas, one or more objects did already exist in odb""" - count = 0 - for sha in sha_iter: - if odb.has_object(sha): - continue - # END check object existance - - ostream = self.stream(sha) - # compressed data including header - sio = StringIO(ostream.stream.data()) - istream = IStream(ostream.type, ostream.size, sio, sha) - - odb.store(istream) - count += 1 - # END for each sha - return count - #} END interface + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError: + raise BadObject(sha) + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.iterkeys() + + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + **Note:** the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existance + + ostream = self.stream(sha) + # compressed data including header + sio = StringIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 4c9d0b919..928731937 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -4,18 +4,18 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" from base import ( - FileDBBase, - ObjectDBR, - CachingDB - ) + FileDBBase, + ObjectDBR, + CachingDB + ) from gitdb.util import LazyMixin from gitdb.exc import ( - BadObject, - UnsupportedOperation, - AmbiguousObjectName - ) + BadObject, + UnsupportedOperation, + AmbiguousObjectName + ) from gitdb.pack import PackEntity @@ -28,182 +28,182 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): - """A database operating on a set of object packs""" - - # sort the priority list every N queries - # Higher values are better, performance tests don't show this has - # any effect, but it should have one - _sort_interval = 500 - - def __init__(self, root_path): - super(PackedDB, self).__init__(root_path) - # list of lists with three items: - # * hits - number of times the pack was hit with a request - # * entity - Pack entity instance - # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query - # self._entities = list() # lazy loaded list - self._hit_count = 0 # amount of hits - self._st_mtime = 0 # last modification data of our root path - - def _set_cache_(self, attr): - if attr == '_entities': - self._entities = list() - self.update_cache(force=True) - # END handle entities initialization - - def _sort_entities(self): - self._entities.sort(key=lambda l: l[0], reverse=True) - - def _pack_info(self, sha): - """:return: tuple(entity, index) for an item at the given sha - :param sha: 20 or 40 byte sha - :raise BadObject: - **Note:** This method is not thread-safe, but may be hit in multi-threaded - operation. The worst thing that can happen though is a counter that - was not incremented, or the list being in wrong order. So we safe - the time for locking here, lets see how that goes""" - # presort ? - if self._hit_count % self._sort_interval == 0: - self._sort_entities() - # END update sorting - - for item in self._entities: - index = item[2](sha) - if index is not None: - item[0] += 1 # one hit for you - self._hit_count += 1 # general hit count - return (item[1], index) - # END index found in pack - # END for each item - - # no hit, see whether we have to update packs - # NOTE: considering packs don't change very often, we safe this call - # and leave it to the super-caller to trigger that - raise BadObject(sha) - - #{ Object DB Read - - def has_object(self, sha): - try: - self._pack_info(sha) - return True - except BadObject: - return False - # END exception handling - - def info(self, sha): - entity, index = self._pack_info(sha) - return entity.info_at_index(index) - - def stream(self, sha): - entity, index = self._pack_info(sha) - return entity.stream_at_index(index) - - def sha_iter(self): - sha_list = list() - for entity in self.entities(): - index = entity.index() - sha_by_index = index.sha - for index in xrange(index.size()): - yield sha_by_index(index) - # END for each index - # END for each entity - - def size(self): - sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes, 0) - - #} END object db read - - #{ object db write - - def store(self, istream): - """Storing individual objects is not feasible as a pack is designed to - hold multiple objects. Writing or rewriting packs for single objects is - inefficient""" - raise UnsupportedOperation() - - def store_async(self, reader): - # TODO: add ObjectDBRW before implementing this - raise NotImplementedError() - - #} END object db write - - - #{ Interface - - def update_cache(self, force=False): - """ - Update our cache with the acutally existing packs on disk. Add new ones, - and remove deleted ones. We keep the unchanged ones - - :param force: If True, the cache will be updated even though the directory - does not appear to have changed according to its modification timestamp. - :return: True if the packs have been updated so there is new information, - False if there was no change to the pack database""" - stat = os.stat(self.root_path()) - if not force and stat.st_mtime <= self._st_mtime: - return False - # END abort early on no change - self._st_mtime = stat.st_mtime - - # packs are supposed to be prefixed with pack- by git-convention - # get all pack files, figure out what changed - pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) - our_pack_files = set(item[1].pack().path() for item in self._entities) - - # new packs - for pack_file in (pack_files - our_pack_files): - # init the hit-counter/priority with the size, a good measure for hit- - # probability. Its implemented so that only 12 bytes will be read - entity = PackEntity(pack_file) - self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) - # END for each new packfile - - # removed packs - for pack_file in (our_pack_files - pack_files): - del_index = -1 - for i, item in enumerate(self._entities): - if item[1].pack().path() == pack_file: - del_index = i - break - # END found index - # END for each entity - assert del_index != -1 - del(self._entities[del_index]) - # END for each removed pack - - # reinitialize prioritiess - self._sort_entities() - return True - - def entities(self): - """:return: list of pack entities operated upon by this database""" - return [ item[1] for item in self._entities ] - - def partial_to_complete_sha(self, partial_binsha, canonical_length): - """:return: 20 byte sha as inferred by the given partial binary sha - :param partial_binsha: binary sha with less than 20 bytes - :param canonical_length: length of the corresponding canonical representation. - It is required as binary sha's cannot display whether the original hex sha - had an odd or even number of characters - :raise AmbiguousObjectName: - :raise BadObject: """ - candidate = None - for item in self._entities: - item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) - if item_index is not None: - sha = item[1].index().sha(item_index) - if candidate and candidate != sha: - raise AmbiguousObjectName(partial_binsha) - candidate = sha - # END handle full sha could be found - # END for each entity - - if candidate: - return candidate - - # still not found ? - raise BadObject(partial_binsha) - - #} END interface + """A database operating on a set of object packs""" + + # sort the priority list every N queries + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 + + def __init__(self, root_path): + super(PackedDB, self).__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + if attr == '_entities': + self._entities = list() + self.update_cache(force=True) + # END handle entities initialization + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + **Note:** This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) + + #{ Object DB Read + + def has_object(self, sha): + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling + + def info(self, sha): + entity, index = self._pack_info(sha) + return entity.info_at_index(index) + + def stream(self, sha): + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) + + def sha_iter(self): + sha_list = list() + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes, 0) + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + def store_async(self, reader): + # TODO: add ObjectDBRW before implementing this + raise NotImplementedError() + + #} END object db write + + + #{ Interface + + def update_cache(self, force=False): + """ + Update our cache with the acutally existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = set(item[1].pack().path() for item in self._entities) + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [ item[1] for item in self._entities ] + + def partial_to_complete_sha(self, partial_binsha, canonical_length): + """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) + + #} END interface diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 898984323..60004a77a 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -3,77 +3,77 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - CompoundDB, - ) + CompoundDB, + ) import os __all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): - """A database consisting of database referred to in a file""" - - # Configuration - # Specifies the object database to use for the paths found in the alternates - # file. If None, it defaults to the GitDB - ObjectDBCls = None - - def __init__(self, ref_file): - super(ReferenceDB, self).__init__() - self._ref_file = ref_file - - def _set_cache_(self, attr): - if attr == '_dbs': - self._dbs = list() - self._update_dbs_from_ref_file() - else: - super(ReferenceDB, self)._set_cache_(attr) - # END handle attrs - - def _update_dbs_from_ref_file(self): - dbcls = self.ObjectDBCls - if dbcls is None: - # late import - from git import GitDB - dbcls = GitDB - # END get db type - - # try to get as many as possible, don't fail if some are unavailable - ref_paths = list() - try: - ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] - except (OSError, IOError): - pass - # END handle alternates - - ref_paths_set = set(ref_paths) - cur_ref_paths_set = set(db.root_path() for db in self._dbs) - - # remove existing - for path in (cur_ref_paths_set - ref_paths_set): - for i, db in enumerate(self._dbs[:]): - if db.root_path() == path: - del(self._dbs[i]) - continue - # END del matching db - # END for each path to remove - - # add new - # sort them to maintain order - added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) - for path in added_paths: - try: - db = dbcls(path) - # force an update to verify path - if isinstance(db, CompoundDB): - db.databases() - # END verification - self._dbs.append(db) - except Exception, e: - # ignore invalid paths or issues - pass - # END for each path to add - - def update_cache(self, force=False): - # re-read alternates and update databases - self._update_dbs_from_ref_file() - return super(ReferenceDB, self).update_cache(force) + """A database consisting of database referred to in a file""" + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super(ReferenceDB, self).__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + else: + super(ReferenceDB, self)._set_cache_(attr) + # END handle attrs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + except (OSError, IOError): + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = set(db.root_path() for db in self._dbs) + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception, e: + # ignore invalid paths or issues + pass + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super(ReferenceDB, self).update_cache(force) diff --git a/gitdb/exc.py b/gitdb/exc.py index e087047b4..7180fb586 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -6,27 +6,27 @@ from util import to_hex_sha class ODBError(Exception): - """All errors thrown by the object database""" - + """All errors thrown by the object database""" + class InvalidDBRoot(ODBError): - """Thrown if an object database cannot be initialized at the given path""" - + """Thrown if an object database cannot be initialized at the given path""" + class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the - failed sha""" - - def __str__(self): - return "BadObject: %s" % to_hex_sha(self.args[0]) - + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) + class ParseError(ODBError): - """Thrown if the parsing of a file failed due to an invalid format""" + """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): - """Thrown if a possibly shortened name does not uniquely represent a single object - in the database""" + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" class BadObjectType(ODBError): - """The object had an unsupported type""" + """The object had an unsupported type""" class UnsupportedOperation(ODBError): - """Thrown if the given operation cannot be supported by the object database""" + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/ext/async b/gitdb/ext/async index 039c1d5c2..571412931 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 039c1d5c26bc2ceaa9e55082efae2068d9873e45 +Subproject commit 571412931829200aff06a44b9c5524e122e524e9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 360a8956f..1b3ab5598 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 360a8956fe73a0a96315e946f52737569d990369 +Subproject commit 1b3ab5598e93369282502d049d64cb2ca12839cb diff --git a/gitdb/fun.py b/gitdb/fun.py index 66130ebee..c1e73e895 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -7,8 +7,8 @@ it into c later, if required""" from exc import ( - BadObjectType - ) + BadObjectType + ) from util import zlib decompressobj = zlib.decompressobj @@ -23,655 +23,655 @@ REF_DELTA = 7 delta_types = (OFS_DELTA, REF_DELTA) -type_id_to_type_map = { - 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", - 5 : "", # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA - } +type_id_to_type_map = { + 0 : "", # EXT 1 + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA + } type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA - ) + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA + ) # used when dealing with larger streams chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures def _set_delta_rbound(d, size): - """Truncate the given delta to the given size - :param size: size relative to our target offset, may not be 0, must be smaller or equal - to our size - :return: d""" - d.ts = size - - # NOTE: data is truncated automatically when applying the delta - # MUST NOT DO THIS HERE - return d - + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size + :return: d""" + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE + return d + def _move_delta_lbound(d, bytes): - """Move the delta by the given amount of bytes, reducing its size so that its - right bound stays static - :param bytes: amount of bytes to move, must be smaller than delta size - :return: d""" - if bytes == 0: - return - - d.to += bytes - d.so += bytes - d.ts -= bytes - if d.data is not None: - d.data = d.data[bytes:] - # END handle data - - return d - + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return + + d.to += bytes + d.so += bytes + d.ts -= bytes + if d.data is not None: + d.data = d.data[bytes:] + # END handle data + + return d + def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data) - + return DeltaChunk(src.to, src.ts, src.so, src.data) + def delta_chunk_apply(dc, bbuf, write): - """Apply own data to the target buffer - :param bbuf: buffer providing source bytes for copy operations - :param write: write method to call with data to write""" - if dc.data is None: - # COPY DATA FROM SOURCE - write(buffer(bbuf, dc.so, dc.ts)) - else: - # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? - # Considering data can be larger than 127 bytes now, it should be worth it - if dc.ts < len(dc.data): - write(dc.data[:dc.ts]) - else: - write(dc.data) - # END handle truncation - # END handle chunk mode + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(buffer(bbuf, dc.so, dc.ts)) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode class DeltaChunk(object): - """Represents a piece of a delta, it can either add new data, or copy existing - one from a source buffer""" - __slots__ = ( - 'to', # start offset in the target buffer in bytes - 'ts', # size of this chunk in the target buffer in bytes - 'so', # start offset in the source buffer in bytes or None - 'data', # chunk of bytes to be added to the target buffer, - # DeltaChunkList to use as base, or None - ) - - def __init__(self, to, ts, so, data): - self.to = to - self.ts = ts - self.so = so - self.data = data + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data', # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data - def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") - - #{ Interface - - def rbound(self): - return self.to + self.ts - - def has_data(self): - """:return: True if the instance has data to add to the target stream""" - return self.data is not None - - #} END interface + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + + #{ Interface + + def rbound(self): + return self.to + self.ts + + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is not None + + #} END interface def _closest_index(dcl, absofs): - """:return: index at which the given absofs should be inserted. The index points - to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs. - **Note:** global method for performance only, it belongs to DeltaChunkList""" - lo = 0 - hi = len(dcl) - while lo < hi: - mid = (lo + hi) / 2 - dc = dcl[mid] - if dc.to > absofs: - hi = mid - elif dc.rbound() > absofs or dc.to == absofs: - return mid - else: - lo = mid + 1 - # END handle bound - # END for each delta absofs - return len(dcl)-1 - + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs. + **Note:** global method for performance only, it belongs to DeltaChunkList""" + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound + # END for each delta absofs + return len(dcl)-1 + def delta_list_apply(dcl, bbuf, write): - """Apply the chain's changes and write the final result using the passed - write function. - :param bbuf: base buffer containing the base of all deltas contained in this - list. It will only be used if the chunk in question does not have a base - chain. - :param write: function taking a string of bytes to write to the output""" - for dc in dcl: - delta_chunk_apply(dc, bbuf, write) - # END for each dc + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc def delta_list_slice(dcl, absofs, size, ndcl): - """:return: Subsection of this list at the given absolute offset, with the given - size in bytes. - :return: None""" - cdi = _closest_index(dcl, absofs) # delta start index - cd = dcl[cdi] - slen = len(dcl) - lappend = ndcl.append - - if cd.to != absofs: - tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) - _move_delta_lbound(tcd, absofs - cd.to) - tcd.ts = min(tcd.ts, size) - lappend(tcd) - size -= tcd.ts - cdi += 1 - # END lbound overlap handling - - while cdi < slen and size: - # are we larger than the current block - cd = dcl[cdi] - if cd.ts <= size: - lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) - size -= cd.ts - else: - tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) - tcd.ts = size - lappend(tcd) - size -= tcd.ts - break - # END hadle size - cdi += 1 - # END for each chunk - - + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: None""" + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + lappend = ndcl.append + + if cd.to != absofs: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + _move_delta_lbound(tcd, absofs - cd.to) + tcd.ts = min(tcd.ts, size) + lappend(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) + size -= cd.ts + else: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size + lappend(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + class DeltaChunkList(list): - """List with special functionality to deal with DeltaChunks. - There are two types of lists we represent. The one was created bottom-up, working - towards the latest delta, the other kind was created top-down, working from the - latest delta down to the earliest ancestor. This attribute is queryable - after all processing with is_reversed.""" - - __slots__ = tuple() - - def rbound(self): - """:return: rightmost extend in bytes, absolute""" - if len(self) == 0: - return 0 - return self[-1].rbound() - - def lbound(self): - """:return: leftmost byte at which this chunklist starts""" - if len(self) == 0: - return 0 - return self[0].to - - def size(self): - """:return: size of bytes as measured by our delta chunks""" - return self.rbound() - self.lbound() - - def apply(self, bbuf, write): - """Only used by public clients, internally we only use the global routines - for performance""" - return delta_list_apply(self, bbuf, write) - - def compress(self): - """Alter the list to reduce the amount of nodes. Currently we concatenate - add-chunks - :return: self""" - slen = len(self) - if slen < 2: - return self - i = 0 - slen_orig = slen - - first_data_index = None - while i < slen: - dc = self[i] - i += 1 - if dc.data is None: - if first_data_index is not None and i-2-first_data_index > 1: - #if first_data_index is not None: - nd = StringIO() # new data - so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i-1): - xdc = self[x] - nd.write(xdc.data[:xdc.ts]) - # END collect data - - del(self[first_data_index:i-1]) - buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) - - slen = len(self) - i = first_data_index + 1 - - # END concatenate data - first_data_index = None - continue - # END skip non-data chunks - - if first_data_index is None: - first_data_index = i-1 - # END iterate list - - #if slen_orig != len(self): - # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) - return self - - def check_integrity(self, target_size=-1): - """Verify the list has non-overlapping chunks only, and the total size matches - target_size - :param target_size: if not -1, the total size of the chain must be target_size - :raise AssertionError: if the size doen't match""" - if target_size > -1: - assert self[-1].rbound() == target_size - assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size - # END target size verification - - if len(self) < 2: - return - - # check data - for dc in self: - assert dc.ts > 0 - if dc.has_data(): - assert len(dc.data) >= dc.ts - # END for each dc - - left = islice(self, 0, len(self)-1) - right = iter(self) - right.next() - # this is very pythonic - we might have just use index based access here, - # but this could actually be faster - for lft,rgt in izip(left, right): - assert lft.rbound() == rgt.to - assert lft.to + lft.ts == rgt.to - # END for each pair - + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def apply(self, bbuf, write): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write) + + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + slen_orig = slen + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if dc.data is None: + if first_data_index is not None and i-2-first_data_index > 1: + #if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in xrange(first_data_index, i-1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i-1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i-1 + # END iterate list + + #if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doen't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + assert dc.ts > 0 + if dc.has_data(): + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self)-1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft,rgt in izip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + class TopdownDeltaChunkList(DeltaChunkList): - """Represents a list which is generated by feeding its ancestor streams one by - one""" - __slots__ = tuple() - - def connect_with_next_base(self, bdcl): - """Connect this chain with the next level of our base delta chunklist. - The goal in this game is to mark as many of our chunks rigid, hence they - cannot be changed by any of the upcoming bases anymore. Once all our - chunks are marked like that, we can stop all processing - :param bdcl: data chunk list being one of our bases. They must be fed in - consequtively and in order, towards the earliest ancestor delta - :return: True if processing was done. Use it to abort processing of - remaining streams if False is returned""" - nfc = 0 # number of frozen chunks - dci = 0 # delta chunk index - slen = len(self) # len of self - ccl = list() # temporary list - while dci < slen: - dc = self[dci] - dci += 1 - - # all add-chunks which are already topmost don't need additional processing - if dc.data is not None: - nfc += 1 - continue - # END skip add chunks - - # copy chunks - # integrate the portion of the base list into ourselves. Lists - # dont support efficient insertion ( just one at a time ), but for now - # we live with it. Internally, its all just a 32/64bit pointer, and - # the portions of moved memory should be smallish. Maybe we just rebuild - # ourselves in order to reduce the amount of insertions ... - del(ccl[:]) - delta_list_slice(bdcl, dc.so, dc.ts, ccl) - - # move the target bounds into place to match with our chunk - ofs = dc.to - dc.so - for cdc in ccl: - cdc.to += ofs - # END update target bounds - - if len(ccl) == 1: - self[dci-1] = ccl[0] - else: - # maybe try to compute the expenses here, and pick the right algorithm - # It would normally be faster than copying everything physically though - # TODO: Use a deque here, and decide by the index whether to extend - # or extend left ! - post_dci = self[dci:] - del(self[dci-1:]) # include deletion of dc - self.extend(ccl) - self.extend(post_dci) - - slen = len(self) - dci += len(ccl)-1 # deleted dc, added rest - - # END handle chunk replacement - # END for each chunk - - if nfc == slen: - return False - # END handle completeness - return True - - + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = tuple() + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consequtively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams if False is returned""" + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + ccl = list() # temporary list + while dci < slen: + dc = self[dci] + dci += 1 + + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + if len(ccl) == 1: + self[dci-1] = ccl[0] + else: + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci-1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl)-1 # deleted dc, added rest + + # END handle chunk replacement + # END for each chunk + + if nfc == slen: + return False + # END handle completeness + return True + + #} END structures #{ Routines def is_loose_object(m): - """ - :return: True the file contained in memory map m appears to be a loose object. - Only the first two bytes are needed""" - b0, b1 = map(ord, m[:2]) - word = (b0 << 8) + b1 - return b0 == 0x78 and (word % 31) == 0 + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 def loose_object_header_info(m): - """ - :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the - object as well as its uncompressed size in bytes. - :param m: memory map from which to read the compressed object data""" - decompress_size = 8192 # is used in cgit as well - hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0")].split(" ") - return type_name, int(size) - + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find("\0")].split(" ") + return type_name, int(size) + def pack_object_header_info(data): - """ - :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) - The type_id should be interpreted according to the ``type_id_to_type_map`` map - The byte-offset specifies the start of the actual zlib compressed datastream - :param m: random-access memory, like a string or memory map""" - c = ord(data[0]) # first byte - i = 1 # next char to read - type_id = (c >> 4) & 7 # numeric type - size = c & 15 # starting size - s = 4 # starting bit-shift size - while c & 0x80: - c = ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop - return (type_id, size, i) + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" + c = ord(data[0]) # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): - """ - :return: string defining the pack header comprised of the object type - and its incompressed size in bytes - - :param obj_type: pack type_id of the object - :param obj_size: uncompressed size in bytes of the following object stream""" - c = 0 # 1 byte - hdr = str() # output string + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = str() # output string - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - #END until size is consumed - hdr += chr(c) - return hdr - + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + return hdr + def msb_size(data, offset=0): - """ - :return: tuple(read_bytes, size) read the msb size from the given random - access data starting at the given byte offset""" - size = 0 - i = 0 - l = len(data) - hit_msb = False - while i < l: - c = ord(data[i+offset]) - size |= (c & 0x7f) << i*7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range - if not hit_msb: - raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size - + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i+offset, size + def loose_object_header(type, size): - """ - :return: string representing the loose object header, which is immediately - followed by the content stream of size 'size'""" - return "%s %i\0" % (type, size) - + """ + :return: string representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return "%s %i\0" % (type, size) + def write_object(type, size, read, write, chunk_size=chunk_size): - """ - Write the object as identified by type, size and source_stream into the - target_stream - - :param type: type string of the object - :param size: amount of bytes to write from source_stream - :param read: read method of a stream providing the content data - :param write: write method of the output stream - :param close_target_stream: if True, the target stream will be closed when - the routine exits, even if an error is thrown - :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" - tbw = 0 # total num bytes written - - # WRITE HEADER: type SP size NULL - tbw += write(loose_object_header(type, size)) - tbw += stream_copy(read, write, size, chunk_size) - - return tbw + """ + Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write(loose_object_header(type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw def stream_copy(read, write, size, chunk_size): - """ - Copy a stream up to size bytes using the provided read and write methods, - in chunks of chunk_size - - **Note:** its much like stream_copy utility, but operates just using methods""" - dbw = 0 # num data bytes written - - # WRITE ALL DATA UP TO SIZE - while True: - cs = min(chunk_size, size-dbw) - # NOTE: not all write methods return the amount of written bytes, like - # mmap.write. Its bad, but we just deal with it ... perhaps its not - # even less efficient - # data_len = write(read(cs)) - # dbw += data_len - data = read(cs) - data_len = len(data) - dbw += data_len - write(data) - if data_len < cs or dbw == size: - break - # END check for stream end - # END duplicate data - return dbw - + """ + Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + + **Note:** its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size-dbw) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) + dbw += data_len + write(data) + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + def connect_deltas(dstreams): - """ - Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks - - :param dstreams: iterable of delta stream objects, the delta to be applied last - comes first, then all its ancestors in order - :return: DeltaChunkList, containing all operations to apply""" - tdcl = None # topmost dcl - - dcl = tdcl = TopdownDeltaChunkList() - for dsi, ds in enumerate(dstreams): - # print "Stream", dsi - db = ds.read() - delta_buf_size = ds.size - - # read header - i, base_size = msb_size(db) - i, target_size = msb_size(db, i) - - # interpret opcodes - tbw = 0 # amount of target bytes written - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > base_size): - break - - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) - tbw += cp_size - elif c: - # NOTE: in C, the data chunks should probably be concatenated here. - # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) - i += c - tbw += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - - dcl.compress() - - # merge the lists ! - if dsi > 0: - if not tdcl.connect_with_next_base(dcl): - break - # END handle merge - - # prepare next base - dcl = DeltaChunkList() - # END for each delta stream - - return tdcl - + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order + :return: DeltaChunkList, containing all operations to apply""" + tdcl = None # topmost dcl + + dcl = tdcl = TopdownDeltaChunkList() + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi + db = ds.read() + delta_buf_size = ds.size + + # read header + i, base_size = msb_size(db) + i, target_size = msb_size(db, i) + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > base_size): + break + + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.compress() + + # merge the lists ! + if dsi > 0: + if not tdcl.connect_with_next_base(dcl): + break + # END handle merge + + # prepare next base + dcl = DeltaChunkList() + # END for each delta stream + + return tdcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): - """ - Apply data from a delta buffer using a source buffer to the target file - - :param src_buf: random access data from which the delta was created - :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes - :param delta_buf: random access delta data - :param write: write method taking a chunk of bytes - - **Note:** transcribed to python from the similar routine in patch-delta.c""" - i = 0 - db = delta_buf - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i+c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - - # yes, lets use the exact same error message that git uses :) - assert i == delta_buf_size, "delta replay has gone wild" - - + """ + Apply data from a delta buffer using a source buffer to the target file + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf: random access delta data + :param write: write method taking a chunk of bytes + + **Note:** transcribed to python from the similar routine in patch-delta.c""" + i = 0 + db = delta_buf + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + + def is_equal_canonical_sha(canonical_length, match, sha1): - """ - :return: True if the given lhs and rhs 20 byte binary shas - The comparison will take the canonical_length of the match sha into account, - hence the comparison will only use the last 4 bytes for uneven canonical representations - :param match: less than 20 byte sha - :param sha1: 20 byte sha""" - binary_length = canonical_length/2 - if match[:binary_length] != sha1[:binary_length]: - return False - - if canonical_length - binary_length and \ - (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: - return False - # END handle uneven canonnical length - return True - + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length/2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + #} END routines try: - # raise ImportError; # DEBUG - from _perf import connect_deltas + # raise ImportError; # DEBUG + from _perf import connect_deltas except ImportError: - pass + pass diff --git a/gitdb/pack.py b/gitdb/pack.py index c6d1cc313..48121f026 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -4,59 +4,59 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) + BadObject, + UnsupportedOperation, + ParseError + ) from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, + ) from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size + ) try: - from _perf import PackIndexFile_sha_to_index + from _perf import PackIndexFile_sha_to_index except ImportError: - pass + pass # END try c module -from base import ( # Amazing ! - OInfo, - OStream, - OPackInfo, - OPackStream, - ODeltaStream, - ODeltaPackInfo, - ODeltaPackStream, - ) +from base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, + ) from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer + ) from struct import ( - pack, - unpack, - ) + pack, + unpack, + ) from binascii import crc32 @@ -70,943 +70,943 @@ - + #{ Utilities def pack_object_at(cursor, offset, as_stream): - """ - :return: Tuple(abs_data_offset, PackInfo|PackStream) - an object of the correct type according to the type_id of the object. - If as_stream is True, the object will contain a stream, allowing the - data to be read decompressed. - :param data: random accessable data containing all required information - :parma offset: offset in to the data at which the object information is located - :param as_stream: if True, a stream object will be returned that can read - the data, otherwise you receive an info object only""" - data = cursor.use_region(offset).buffer() - type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) - total_rela_offset = None # set later, actual offset until data stream begins - delta_info = None - - # OFFSET DELTA - if type_id == OFS_DELTA: - i = data_rela_offset - c = ord(data[i]) - i += 1 - delta_offset = c & 0x7f - while c & 0x80: - c = ord(data[i]) - i += 1 - delta_offset += 1 - delta_offset = (delta_offset << 7) + (c & 0x7f) - # END character loop - delta_info = delta_offset - total_rela_offset = i - # REF DELTA - elif type_id == REF_DELTA: - total_rela_offset = data_rela_offset+20 - delta_info = data[data_rela_offset:total_rela_offset] - # BASE OBJECT - else: - # assume its a base object - total_rela_offset = data_rela_offset - # END handle type id - - abs_data_offset = offset + total_rela_offset - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) - if delta_info is None: - return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) - else: - return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) - else: - if delta_info is None: - return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) - else: - return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) - # END handle info - # END handle stream + """ + :return: Tuple(abs_data_offset, PackInfo|PackStream) + an object of the correct type according to the type_id of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. + :param data: random accessable data containing all required information + :parma offset: offset in to the data at which the object information is located + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only""" + data = cursor.use_region(offset).buffer() + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA + if type_id == OFS_DELTA: + i = data_rela_offset + c = ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: + c = ord(data[i]) + i += 1 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) + # END character loop + delta_info = delta_offset + total_rela_offset = i + # REF DELTA + elif type_id == REF_DELTA: + total_rela_offset = data_rela_offset+20 + delta_info = data[data_rela_offset:total_rela_offset] + # BASE OBJECT + else: + # assume its a base object + total_rela_offset = data_rela_offset + # END handle type id + + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + if delta_info is None: + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) + else: + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) + else: + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream def write_stream_to_pack(read, write, zstream, base_crc=None): - """Copy a stream as read from read function, zip it, and write the result. - Count the number of written bytes and return it - :param base_crc: if not None, the crc will be the base for all compressed data - we consecutively write and generate a crc32 from. If None, no crc will be generated - :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc - was false""" - br = 0 # bytes read - bw = 0 # bytes written - want_crc = base_crc is not None - crc = 0 - if want_crc: - crc = base_crc - #END initialize crc - - while True: - chunk = read(chunk_size) - br += len(chunk) - compressed = zstream.compress(chunk) - bw += len(compressed) - write(compressed) # cannot assume return value - - if want_crc: - crc = crc32(compressed, crc) - #END handle crc - - if len(chunk) != chunk_size: - break - #END copy loop - - compressed = zstream.flush() - bw += len(compressed) - write(compressed) - if want_crc: - crc = crc32(compressed, crc) - #END handle crc - - return (br, bw, crc) + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + want_crc = base_crc is not None + crc = 0 + if want_crc: + crc = base_crc + #END initialize crc + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + if len(chunk) != chunk_size: + break + #END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + return (br, bw, crc) #} END utilities class IndexWriter(object): - """Utility to cache index information, allowing to write all information later - in one go to the given stream - **Note:** currently only writes v2 indices""" - __slots__ = '_objs' - - def __init__(self): - self._objs = list() - - def append(self, binsha, crc, offset): - """Append one piece of object information""" - self._objs.append((binsha, crc, offset)) - - def write(self, pack_sha, write): - """Write the index file using the given write method - :param pack_sha: binary sha over the whole pack that we index - :return: sha1 binary sha over all index file contents""" - # sort for sha1 hash - self._objs.sort(key=lambda o: o[0]) - - sha_writer = FlexibleSha1Writer(write) - sha_write = sha_writer.write - sha_write(PackIndexFile.index_v2_signature) - sha_write(pack(">L", PackIndexFile.index_version_default)) - - # fanout - tmplist = list((0,)*256) # fanout or list with 64 bit offsets - for t in self._objs: - tmplist[ord(t[0][0])] += 1 - #END prepare fanout - for i in xrange(255): - v = tmplist[i] - sha_write(pack('>L', v)) - tmplist[i+1] += v - #END write each fanout entry - sha_write(pack('>L', tmplist[255])) - - # sha1 ordered - # save calls, that is push them into c - sha_write(''.join(t[0] for t in self._objs)) - - # crc32 - for t in self._objs: - sha_write(pack('>L', t[1]&0xffffffff)) - #END for each crc - - tmplist = list() - # offset 32 - for t in self._objs: - ofs = t[2] - if ofs > 0x7fffffff: - tmplist.append(ofs) - ofs = 0x80000000 + len(tmplist)-1 - #END hande 64 bit offsets - sha_write(pack('>L', ofs&0xffffffff)) - #END for each offset - - # offset 64 - for ofs in tmplist: - sha_write(pack(">Q", ofs)) - #END for each offset - - # trailer - assert(len(pack_sha) == 20) - sha_write(pack_sha) - sha = sha_writer.sha(as_hex=False) - write(sha) - return sha - - + """Utility to cache index information, allowing to write all information later + in one go to the given stream + **Note:** currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_sha, write): + """Write the index file using the given write method + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,)*256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[ord(t[0][0])] += 1 + #END prepare fanout + for i in xrange(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i+1] += v + #END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1]&0xffffffff)) + #END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist)-1 + #END hande 64 bit offsets + sha_write(pack('>L', ofs&0xffffffff)) + #END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + #END for each offset + + # trailer + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha + + class PackIndexFile(LazyMixin): - """A pack index provides offsets into the corresponding pack, allowing to find - locations for offsets faster.""" - - # Dont use slots as we dynamically bind functions for each version, need a dict for this - # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', - # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') - # used in v2 indices - _sha_list_offset = 8 + 1024 - index_v2_signature = '\377tOc' - index_version_default = 2 + # used in v2 indices + _sha_list_offset = 8 + 1024 + index_v2_signature = '\377tOc' + index_version_default = 2 - def __init__(self, indexpath): - super(PackIndexFile, self).__init__() - self._indexpath = indexpath - - def _set_cache_(self, attr): - if attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-40:-20] - elif attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-20:] - elif attr == "_cursor": - # Note: We don't lock the file when reading as we cannot be sure - # that we can actually write to the location - it could be a read-only - # alternate for instance - self._cursor = mman.make_cursor(self._indexpath).use_region() - # We will assume that the index will always fully fit into memory ! - if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): - raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) - #END assert window size - else: - # now its time to initialize everything - if we are here, someone wants - # to access the fanout table or related properties - - # CHECK VERSION - mmap = self._cursor.map() - self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 - if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] - assert version_id == self._version, "Unsupported index version: %i" % version_id - # END assert version - - # SETUP FUNCTIONS - # setup our functions according to the actual version - for fname in ('entry', 'offset', 'sha', 'crc'): - setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) - # END for each function to initialize - - - # INITIALIZE DATA - # byte offset is 8 if version is 2, 0 otherwise - self._initialize() - # END handle attributes - + def __init__(self, indexpath): + super(PackIndexFile, self).__init__() + self._indexpath = indexpath + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-40:-20] + elif attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-20:] + elif attr == "_cursor": + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) + #END assert window size + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", mmap, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + - #{ Access V1 - - def _entry_v1(self, i): - """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - - def _offset_v1(self, i): - """see ``_offset_v2``""" - return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - - def _sha_v1(self, i): - """see ``_sha_v2``""" - base = 1024 + (i*24)+4 - return self._cursor.map()[base:base+20] - - def _crc_v1(self, i): - """unsupported""" - return 0 - - #} END access V1 - - #{ Access V2 - def _entry_v2(self, i): - """:return: tuple(offset, binsha, crc)""" - return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - - def _offset_v2(self, i): - """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only - be returned if the pack is larger than 4 GiB, or 2^32""" - offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - - # if the high-bit is set, this indicates that we have to lookup the offset - # in the 64 bit region of the file. The current offset ( lower 31 bits ) - # are the index into it - if offset & 0x80000000: - offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] - # END handle 64 bit offset - - return offset - - def _sha_v2(self, i): - """:return: sha at the given index of this file index instance""" - base = self._sha_list_offset + i * 20 - return self._cursor.map()[base:base+20] - - def _crc_v2(self, i): - """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - - #} END access V2 - - #{ Initialization - - def _initialize(self): - """initialize base data""" - self._fanout_table = self._read_fanout((self._version == 2) * 8) - - if self._version == 2: - self._crc_list_offset = self._sha_list_offset + self.size() * 20 - self._pack_offset = self._crc_list_offset + self.size() * 4 - self._pack_64_offset = self._pack_offset + self.size() * 4 - # END setup base - - def _read_fanout(self, byte_offset): - """Generate a fanout table from our data""" - d = self._cursor.map() - out = list() - append = out.append - for i in range(256): - append(unpack_from('>L', d, byte_offset + i*4)[0]) - # END for each entry - return out - - #} END initialization - - #{ Properties - def version(self): - return self._version - - def size(self): - """:return: amount of objects referred to by this index""" - return self._fanout_table[255] - - def path(self): - """:return: path to the packindexfile""" - return self._indexpath - - def packfile_checksum(self): - """:return: 20 byte sha representing the sha1 hash of the pack file""" - return self._cursor.map()[-40:-20] - - def indexfile_checksum(self): - """:return: 20 byte sha representing the sha1 hash of this index file""" - return self._cursor.map()[-20:] - - def offsets(self): - """:return: sequence of all offsets in the order in which they were written - - **Note:** return value can be random accessed, but may be immmutable""" - if self._version == 2: - # read stream to array, convert to tuple - a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - - # networkbyteorder to something array likes more - if sys.byteorder == 'little': - a.byteswap() - return a - else: - return tuple(self.offset(index) for index in xrange(self.size())) - # END handle version - - def sha_to_index(self, sha): - """ - :return: index usable with the ``offset`` or ``entry`` method, or None - if the sha was not found in this pack index - :param sha: 20 byte sha to lookup""" - first_byte = ord(sha[0]) - get_sha = self.sha - lo = 0 # lower index, the left bound of the bisection - if first_byte != 0: - lo = self._fanout_table[first_byte-1] - hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - - # bisect until we have the sha - while lo < hi: - mid = (lo + hi) / 2 - c = cmp(sha, get_sha(mid)) - if c < 0: - hi = mid - elif not c: - return mid - else: - lo = mid + 1 - # END handle midpoint - # END bisect - return None - - def partial_sha_to_index(self, partial_bin_sha, canonical_length): - """ - :return: index as in `sha_to_index` or None if the sha was not found in this - index file - :param partial_bin_sha: an at least two bytes of a partial binary sha - :param canonical_length: lenght of the original hexadecimal representation of the - given partial binary sha - :raise AmbiguousObjectName:""" - if len(partial_bin_sha) < 2: - raise ValueError("Require at least 2 bytes of partial sha") - - first_byte = ord(partial_bin_sha[0]) - get_sha = self.sha - lo = 0 # lower index, the left bound of the bisection - if first_byte != 0: - lo = self._fanout_table[first_byte-1] - hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - - # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest - while lo < hi: - mid = (lo + hi) / 2 - c = cmp(filled_sha, get_sha(mid)) - if c < 0: - hi = mid - elif not c: - # perfect match - lo = mid - break - else: - lo = mid + 1 - # END handle midpoint - # END bisect - - if lo < self.size(): - cur_sha = get_sha(lo) - if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): - next_sha = None - if lo+1 < self.size(): - next_sha = get_sha(lo+1) - if next_sha and next_sha == cur_sha: - raise AmbiguousObjectName(partial_bin_sha) - return lo - # END if we have a match - # END if we found something - return None - - if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr - # accesses - def sha_to_index(self, sha): - return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - - #} END properties - - + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + (i*24)+4 + return self._cursor.map()[base:base+20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._cursor.map()[base:base+20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._cursor.map() + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i*4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + def version(self): + return self._version + + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._cursor.map()[-40:-20] + + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._cursor.map()[-20:] + + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + + **Note:** return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + + # networkbyteorder to something array likes more + if sys.byteorder == 'little': + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in xrange(self.size())) + # END handle version + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = ord(sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + return mid + else: + lo = mid + 1 + # END handle midpoint + # END bisect + return None + + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha + :param canonical_length: lenght of the original hexadecimal representation of the + given partial binary sha + :raise AmbiguousObjectName:""" + if len(partial_bin_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + first_byte = ord(partial_bin_sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # fill the partial to full 20 bytes + filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) + + # find lowest + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(filled_sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + + if lo < self.size(): + cur_sha = get_sha(lo) + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): + next_sha = None + if lo+1 < self.size(): + next_sha = get_sha(lo+1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_bin_sha) + return lo + # END if we have a match + # END if we found something + return None + + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + + #} END properties + + class PackFile(LazyMixin): - """A pack is a file written according to the Version 2 for git packs - - As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be - fine though. - - **Note:** at some point, this might be implemented using streams as well, or - streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that - case""" - - __slots__ = ('_packpath', '_cursor', '_size', '_version') - pack_signature = 0x5041434b # 'PACK' - pack_version_default = 2 - - # offset into our data at which the first object starts - first_object_offset = 3*4 # header bytes - footer_size = 20 # final sha - - def __init__(self, packpath): - self._packpath = packpath - - def _set_cache_(self, attr): - # we fill the whole cache, whichever attribute gets queried first - self._cursor = mman.make_cursor(self._packpath).use_region() - - # read the header information - type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - if type_id != self.pack_signature: - raise ParseError("Invalid pack signature: %i" % type_id) - - def _iter_objects(self, start_offset, as_stream=True): - """Handle the actual iteration of objects within this pack""" - c = self._cursor - content_size = c.file_size() - self.footer_size - cur_offset = start_offset or self.first_object_offset - - null = NullStream() - while cur_offset < content_size: - data_offset, ostream = pack_object_at(c, cur_offset, True) - # scrub the stream to the end - this decompresses the object, but yields - # the amount of compressed bytes we need to get to the next offset - - stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - - # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the - # info object - if as_stream: - ostream.stream.seek(0) - yield ostream - # END until we have read everything - - #{ Pack Information - - def size(self): - """:return: The amount of objects stored in this pack""" - return self._size - - def version(self): - """:return: the version of this pack""" - return self._version - - def data(self): - """ - :return: read-only data of this pack. It provides random access and usually - is a memory map. - :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" - # can use map as we are starting at offset 0. Otherwise we would have to use buffer() - return self._cursor.use_region().map() - - def checksum(self): - """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - - def path(self): - """:return: path to the packfile""" - return self._packpath - #} END pack information - - #{ Pack Specific - - def collect_streams(self, offset): - """ - :return: list of pack streams which are required to build the object - at the given offset. The first entry of the list is the object at offset, - the last one is either a full object, or a REF_Delta stream. The latter - type needs its reference object to be locked up in an ODB to form a valid - delta chain. - If the object at offset is no delta, the size of the list is 1. - :param offset: specifies the first byte of the object within this pack""" - out = list() - c = self._cursor - while True: - ostream = pack_object_at(c, offset, True)[1] - out.append(ostream) - if ostream.type_id == OFS_DELTA: - offset = ostream.pack_offset - ostream.delta_info - else: - # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter - # case someone else has to find it - break - # END handle type - # END while chaining streams - return out + """A pack is a file written according to the Version 2 for git packs + + As we currently use memory maps, it could be assumed that the maximum size of + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_cursor', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 + + # offset into our data at which the first object starts + first_object_offset = 3*4 # header bytes + footer_size = 20 # final sha + + def __init__(self, packpath): + self._packpath = packpath + + def _set_cache_(self, attr): + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + + def _iter_objects(self, start_offset, as_stream=True): + """Handle the actual iteration of objects within this pack""" + c = self._cursor + content_size = c.file_size() - self.footer_size + cur_offset = start_offset or self.first_object_offset + + null = NullStream() + while cur_offset < content_size: + data_offset, ostream = pack_object_at(c, cur_offset, True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream + # END until we have read everything + + #{ Pack Information + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def data(self): + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] + + def path(self): + """:return: path to the packfile""" + return self._packpath + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + If the object at offset is no delta, the size of the list is 1. + :param offset: specifies the first byte of the object within this pack""" + out = list() + c = self._cursor + while True: + ostream = pack_object_at(c, offset, True)[1] + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out - #} END pack specific - - #{ Read-Database like Interface - - def info(self, offset): - """Retrieve information about the object at the given file-absolute offset - - :param offset: byte offset - :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - - def stream(self, offset): - """Retrieve an object at the given file-relative offset as stream along with its information - - :param offset: byte offset - :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - - def stream_iter(self, start_offset=0): - """ - :return: iterator yielding OPackStream compatible instances, allowing - to access the data in the pack directly. - :param start_offset: offset to the first object to iterate. If 0, iteration - starts at the very first object in the pack. - - **Note:** Iterating a pack directly is costly as the datastream has to be decompressed - to determine the bounds between the objects""" - return self._iter_objects(start_offset, as_stream=True) - - #} END Read-Database like Interface - - + #} END pack specific + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] + + def stream_iter(self, start_offset=0): + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + + #} END Read-Database like Interface + + class PackEntity(LazyMixin): - """Combines the PackIndexFile and the PackFile into one, allowing the - actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file - '_pack', # our pack file - '_offset_map' # on demand dict mapping one offset to the next consecutive one - ) - - IndexFileCls = PackIndexFile - PackFileCls = PackFile - - def __init__(self, pack_or_index_path): - """Initialize ourselves with the path to the respective pack or index file""" - basename, ext = os.path.splitext(pack_or_index_path) - self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance - self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - - def _set_cache_(self, attr): - # currently this can only be _offset_map - # TODO: make this a simple sorted offset array which can be bisected - # to find the respective entry, from which we can take a +1 easily - # This might be slower, but should also be much lighter in memory ! - offsets_sorted = sorted(self._index.offsets()) - last_offset = len(self._pack.data()) - self._pack.footer_size - assert offsets_sorted, "Cannot handle empty indices" - - offset_map = None - if len(offsets_sorted) == 1: - offset_map = { offsets_sorted[0] : last_offset } - else: - iter_offsets = iter(offsets_sorted) - iter_offsets_plus_one = iter(offsets_sorted) - iter_offsets_plus_one.next() - consecutive = izip(iter_offsets, iter_offsets_plus_one) - - offset_map = dict(consecutive) - - # the last offset is not yet set - offset_map[offsets_sorted[-1]] = last_offset - # END handle offset amount - self._offset_map = offset_map - - def _sha_to_index(self, sha): - """:return: index for the given sha, or raise""" - index = self._index.sha_to_index(sha) - if index is None: - raise BadObject(sha) - return index - - def _iter_objects(self, as_stream): - """Iterate over all objects in our index and yield their OInfo or OStream instences""" - _sha = self._index.sha - _object = self._object - for index in xrange(self._index.size()): - yield _object(_sha(index), as_stream, index) - # END for each index - - def _object(self, sha, as_stream, index=-1): - """:return: OInfo or OStream object providing information about the given sha - :param index: if not -1, its assumed to be the sha's index in the IndexFile""" - # its a little bit redundant here, but it needs to be efficient - if index < 0: - index = self._sha_to_index(sha) - if sha is None: - sha = self._index.sha(index) - # END assure sha is present ( in output ) - offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) - if as_stream: - if type_id not in delta_types: - packstream = self._pack.stream(offset) - return OStream(sha, packstream.type, packstream.size, packstream.stream) - # END handle non-deltas - - # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, - # we extract it from the delta stream ourselves - streams = self.collect_streams_at_offset(offset) - dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) - else: - if type_id not in delta_types: - return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) - # END handle non-deltas - - # deltas are a little tougher - unpack the first bytes to obtain - # the actual target size, as opposed to the size of the delta data - streams = self.collect_streams_at_offset(offset) - buf = streams[0].read(512) - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - - # collect the streams to obtain the actual object type - if streams[-1].type_id in delta_types: - raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) - # END handle stream - - #{ Read-Database like Interface - - def info(self, sha): - """Retrieve information about the object identified by the given sha - - :param sha: 20 byte sha1 - :raise BadObject: - :return: OInfo instance, with 20 byte sha""" - return self._object(sha, False) - - def stream(self, sha): - """Retrieve an object stream along with its information as identified by the given sha - - :param sha: 20 byte sha1 - :raise BadObject: - :return: OStream instance, with 20 byte sha""" - return self._object(sha, True) + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ( '_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def _set_cache_(self, attr): + # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = { offsets_sorted[0] : last_offset } + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + iter_offsets_plus_one.next() + consecutive = izip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map + + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + + def _iter_objects(self, as_stream): + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + _sha = self._index.sha + _object = self._object + for index in xrange(self._index.size()): + yield _object(_sha(index), as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" + # its a little bit redundant here, but it needs to be efficient + if index < 0: + index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) + offset = self._index.offset(index) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + dstream = DeltaApplyReader.new(streams) + + return ODeltaStream(sha, dstream.type, None, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + return OInfo(sha, streams[-1].type, target_size) + # END handle stream + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OInfo instance, with 20 byte sha""" + return self._object(sha, False) + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OStream instance, with 20 byte sha""" + return self._object(sha, True) - def info_at_index(self, index): - """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" - return self._object(None, False, index) - - def stream_at_index(self, index): - """As ``stream``, but uses a PackIndexFile compatible index to refer to the - object""" - return self._object(None, True, index) - - #} END Read-Database like Interface - - #{ Interface + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) + + #} END Read-Database like Interface + + #{ Interface - def pack(self): - """:return: the underlying pack file instance""" - return self._pack - - def index(self): - """:return: the underlying pack index file instance""" - return self._index - - def is_valid_stream(self, sha, use_crc=False): - """ - Verify that the stream at the given sha is valid. - - :param use_crc: if True, the index' crc is run over the compressed stream of - the object, which is much faster than checking the sha1. It is also - more prone to unnoticed corruption or manipulation. - :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is - a delta, this only verifies that the delta's data is valid, not the - data of the actual undeltified object, as it depends on more than - just this stream. - If False, the object will be decompressed and the sha generated. It must - match the given sha - - :return: True if the stream is valid - :raise UnsupportedOperation: If the index is version 1 only - :raise BadObject: sha was not found""" - if use_crc: - if self._index.version() < 2: - raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") - # END handle index version - - index = self._sha_to_index(sha) - offset = self._index.offset(index) - next_offset = self._offset_map[offset] - crc_value = self._index.crc(index) - - # create the current crc value, on the compressed object data - # Read it in chunks, without copying the data - crc_update = zlib.crc32 - pack_data = self._pack.data() - cur_pos = offset - this_crc_value = 0 - while cur_pos < next_offset: - rbound = min(cur_pos + chunk_size, next_offset) - size = rbound - cur_pos - this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) - cur_pos += size - # END window size loop - - # crc returns signed 32 bit numbers, the AND op forces it into unsigned - # mode ... wow, sneaky, from dulwich. - return (this_crc_value & 0xffffffff) == crc_value - else: - shawriter = Sha1Writer() - stream = self._object(sha, as_stream=True) - # write a loose object, which is the basis for the sha - write_object(stream.type, stream.size, stream.read, shawriter.write) - - assert shawriter.sha(as_hex=False) == sha - return shawriter.sha(as_hex=False) == sha - # END handle crc/sha verification - return True + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """ + Verify that the stream at the given sha is valid. + + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + + index = self._sha_to_index(sha) + offset = self._index.offset(index) + next_offset = self._offset_map[offset] + crc_value = self._index.crc(index) + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() + cur_pos = offset + this_crc_value = 0 + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + cur_pos += size + # END window size loop + + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + assert shawriter.sha(as_hex=False) == sha + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + return True - def info_iter(self): - """ - :return: Iterator over all objects in this pack. The iterator yields - OInfo instances""" - return self._iter_objects(as_stream=False) - - def stream_iter(self): - """ - :return: iterator over all objects in this pack. The iterator yields - OStream instances""" - return self._iter_objects(as_stream=True) - - def collect_streams_at_offset(self, offset): - """ - As the version in the PackFile, but can resolve REF deltas within this pack - For more info, see ``collect_streams`` - - :param offset: offset into the pack file at which the object can be found""" - streams = self._pack.collect_streams(offset) - - # try to resolve the last one if needed. It is assumed to be either - # a REF delta, or a base object, as OFFSET deltas are resolved by the pack - if streams[-1].type_id == REF_DELTA: - stream = streams[-1] - while stream.type_id in delta_types: - if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(stream.delta_info) - if sindex is None: - break - stream = self._pack.stream(self._index.offset(sindex)) - streams.append(stream) - else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who - # would do that ;) ? We can handle it though - stream = self._pack.stream(stream.delta_info) - streams.append(stream) - # END handle ref delta - # END resolve ref streams - # END resolve streams - - return streams - - def collect_streams(self, sha): - """ - As ``PackFile.collect_streams``, but takes a sha instead of an offset. - Additionally, ref_delta streams will be resolved within this pack. - If this is not possible, the stream will be left alone, hence it is adivsed - to check for unresolved ref-deltas and resolve them before attempting to - construct a delta stream. - - :param sha: 20 byte sha1 specifying the object whose related streams you want to collect - :return: list of streams, first being the actual object delta, the last being - a possibly unresolved base object. - :raise BadObject:""" - return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - - @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, - object_count = None, zlib_compression = zlib.Z_BEST_SPEED): - """ - Create a new pack by putting all objects obtained by the object_iterator - into a pack which is written using the pack_write method. - The respective index is produced as well if index_write is not Non. - - :param object_iter: iterator yielding odb output objects - :param pack_write: function to receive strings to write into the pack stream - :param indx_write: if not None, the function writes the index file corresponding - to the pack. - :param object_count: if you can provide the amount of objects in your iteration, - this would be the place to put it. Otherwise we have to pre-iterate and store - all items into a list to get the number, which uses more memory than necessary. - :param zlib_compression: the zlib compression level to use - :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack - and over all contents of the index. If index_write was None, index_binsha will be None - - **Note:** The destination of the write functions is up to the user. It could - be a socket, or a file for instance - - **Note:** writes only undeltified objects""" - objs = object_iter - if not object_count: - if not isinstance(object_iter, (tuple, list)): - objs = list(object_iter) - #END handle list type - object_count = len(objs) - #END handle object - - pack_writer = FlexibleSha1Writer(pack_write) - pwrite = pack_writer.write - ofs = 0 # current offset into the pack file - index = None - wants_index = index_write is not None - - # write header - pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) - ofs += 12 - - if wants_index: - index = IndexWriter() - #END handle index header - - actual_count = 0 - for obj in objs: - actual_count += 1 - crc = 0 - - # object header - hdr = create_pack_object_header(obj.type_id, obj.size) - if index_write: - crc = crc32(hdr) - else: - crc = None - #END handle crc - pwrite(hdr) - - # data stream - zstream = zlib.compressobj(zlib_compression) - ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) - assert(br == obj.size) - if wants_index: - index.append(obj.binsha, crc, ofs) - #END handle index - - ofs += len(hdr) + bw - if actual_count == object_count: - break - #END abort once we are done - #END for each object - - if actual_count != object_count: - raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) - #END count assertion - - # write footer - pack_sha = pack_writer.sha(as_hex = False) - assert len(pack_sha) == 20 - pack_write(pack_sha) - ofs += len(pack_sha) # just for completeness ;) - - index_sha = None - if wants_index: - index_sha = index.write(pack_sha, index_write) - #END handle index - - return pack_sha, index_sha - - @classmethod - def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): - """Create a new on-disk entity comprised of a properly named pack file and a properly named - and corresponding index file. The pack contains all OStream objects contained in object iter. - :param base_dir: directory which is to contain the files - :return: PackEntity instance initialized with the new pack - - **Note:** for more information on the other parameters see the write_pack method""" - pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) - index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) - pack_write = lambda d: os.write(pack_fd, d) - index_write = lambda d: os.write(index_fd, d) - - pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) - os.close(pack_fd) - os.close(index_fd) - - fmt = "pack-%s.%s" - new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) - new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) - os.rename(pack_path, new_pack_path) - os.rename(index_path, new_index_path) - - return cls(new_pack_path) - - - #} END interface + def info_iter(self): + """ + :return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + def collect_streams_at_offset(self, offset): + """ + As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + + @classmethod + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + #END handle list type + object_count = len(objs) + #END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + #END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + crc = 0 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + #END handle crc + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + #END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + #END abort once we are done + #END for each object + + if actual_count != object_count: + raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + #END count assertion + + # write footer + pack_sha = pack_writer.sha(as_hex = False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + + index_sha = None + if wants_index: + index_sha = index.write(pack_sha, index_write) + #END handle index + + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + + **Note:** for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) + + + #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 632213c27..6441b1e1a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -9,684 +9,684 @@ import os from fun import ( - msb_size, - stream_copy, - apply_delta_data, - connect_deltas, - DeltaChunkList, - delta_types - ) + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + DeltaChunkList, + delta_types + ) from util import ( - allocate_memory, - LazyMixin, - make_sha, - write, - close, - zlib - ) + allocate_memory, + LazyMixin, + make_sha, + write, + close, + zlib + ) has_perf_mod = False try: - from _perf import apply_delta as c_apply_delta - has_perf_mod = True + from _perf import apply_delta as c_apply_delta + has_perf_mod = True except ImportError: - pass + pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', - 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', - 'FDStream', 'NullStream') +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams class DecompressMemMapReader(LazyMixin): - """Reads data in chunks from a memory map and decompresses it. The client sees - only the uncompressed data, respective file-like read calls are handling on-demand - buffered decompression accordingly - - A constraint on the total size of bytes is activated, simulating - a logical file within a possibly larger physical memory area - - To read efficiently, you clearly don't want to read individual bytes, instead, - read a few kilobytes at least. - - **Note:** The chunk-size should be carefully selected as it will involve quite a bit - of string copying due to the way the zlib is implemented. Its very wasteful, - hence we try to find a good tradeoff between allocation time and number of - times we actually allocate. An own zlib implementation would be good here - to better support streamed reading - it would only need to keep the mmap - and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', - '_cbr', '_phi') - - max_read_size = 512*1024 # currently unused - - def __init__(self, m, close_on_deletion, size=None): - """Initialize with mmap for stream reading - :param m: must be content data - use new if you have object data and no size""" - self._m = m - self._zip = zlib.decompressobj() - self._buf = None # buffer of decompressed bytes - self._buflen = 0 # length of bytes in buffer - if size is not None: - self._s = size # size of uncompressed data to read in total - self._br = 0 # num uncompressed bytes read - self._cws = 0 # start byte of compression window - self._cwe = 0 # end byte of compression window - self._cbr = 0 # number of compressed bytes read - self._phi = False # is True if we parsed the header info - self._close = close_on_deletion # close the memmap on deletion ? - - def _set_cache_(self, attr): - assert attr == '_s' - # only happens for size, which is a marker to indicate we still - # have to parse the header from the stream - self._parse_header_info() - - def __del__(self): - if self._close: - self._m.close() - # END handle resource freeing - - def _parse_header_info(self): - """If this stream contains object data, parse the header info and skip the - stream to a point where each read will yield object content - - :return: parsed type_string, size""" - # read header - maxb = 512 # should really be enough, cgit uses 8192 I believe - self._s = maxb - hdr = self.read(maxb) - hdrend = hdr.find("\0") - type, size = hdr[:hdrend].split(" ") - size = int(size) - self._s = size - - # adjust internal state to match actual header length that we ignore - # The buffer will be depleted first on future reads - self._br = 0 - hdrend += 1 # count terminating \0 - self._buf = StringIO(hdr[hdrend:]) - self._buflen = len(hdr) - hdrend - - self._phi = True - - return type, size - - #{ Interface - - @classmethod - def new(self, m, close_on_deletion=False): - """Create a new DecompressMemMapReader instance for acting as a read-only stream - This method parses the object header from m and returns the parsed - type and size, as well as the created stream instance. - - :param m: memory map on which to oparate. It must be object data ( header + contents ) - :param close_on_deletion: if True, the memory map will be closed once we are - being deleted""" - inst = DecompressMemMapReader(m, close_on_deletion, 0) - type, size = inst._parse_header_info() - return type, size, inst - - def data(self): - """:return: random access compatible data we are working on""" - return self._m - - def compressed_bytes_read(self): - """ - :return: number of compressed bytes read. This includes the bytes it - took to decompress the header ( if there was one )""" - # ABSTRACT: When decompressing a byte stream, it can be that the first - # x bytes which were requested match the first x bytes in the loosely - # compressed datastream. This is the worst-case assumption that the reader - # does, it assumes that it will get at least X bytes from X compressed bytes - # in call cases. - # The caveat is that the object, according to our known uncompressed size, - # is already complete, but there are still some bytes left in the compressed - # stream that contribute to the amount of compressed bytes. - # How can we know that we are truly done, and have read all bytes we need - # to read ? - # Without help, we cannot know, as we need to obtain the status of the - # decompression. If it is not finished, we need to decompress more data - # until it is finished, to yield the actual number of compressed bytes - # belonging to the decompressed object - # We are using a custom zlib module for this, if its not present, - # we try to put in additional bytes up for decompression if feasible - # and check for the unused_data. - - # Only scrub the stream forward if we are officially done with the - # bytes we were to have. - if self._br == self._s and not self._zip.unused_data: - # manipulate the bytes-read to allow our own read method to coninute - # but keep the window at its current position - self._br = 0 - if hasattr(self._zip, 'status'): - while self._zip.status == zlib.Z_OK: - self.read(mmap.PAGESIZE) - # END scrub-loop custom zlib - else: - # pass in additional pages, until we have unused data - while not self._zip.unused_data and self._cbr != len(self._m): - self.read(mmap.PAGESIZE) - # END scrub-loop default zlib - # END handle stream scrubbing - - # reset bytes read, just to be sure - self._br = self._s - # END handle stream scrubbing - - # unused data ends up in the unconsumed tail, which was removed - # from the count already - return self._cbr - - #} END interface - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Allows to reset the stream to restart reading - :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - - self._zip = zlib.decompressobj() - self._br = self._cws = self._cwe = self._cbr = 0 - if self._phi: - self._phi = False - del(self._s) # trigger header parsing on first access - # END skip header - - def read(self, size=-1): - if size < 1: - size = self._s - self._br - else: - size = min(size, self._s - self._br) - # END clamp size - - if size == 0: - return str() - # END handle depletion - - - # deplete the buffer, then just continue using the decompress object - # which has an own buffer. We just need this to transparently parse the - # header from the zlib stream - dat = str() - if self._buf: - if self._buflen >= size: - # have enough data - dat = self._buf.read(size) - self._buflen -= size - self._br += size - return dat - else: - dat = self._buf.read() # ouch, duplicates data - size -= self._buflen - self._br += self._buflen - - self._buflen = 0 - self._buf = None - # END handle buffer len - # END handle buffer - - # decompress some data - # Abstract: zlib needs to operate on chunks of our memory map ( which may - # be large ), as it will otherwise and always fill in the 'unconsumed_tail' - # attribute which possible reads our whole map to the end, forcing - # everything to be read from disk even though just a portion was requested. - # As this would be a nogo, we workaround it by passing only chunks of data, - # moving the window into the memory map along as we decompress, which keeps - # the tail smaller than our chunk-size. This causes 'only' the chunk to be - # copied once, and another copy of a part of it when it creates the unconsumed - # tail. We have to use it to hand in the appropriate amount of bytes durin g - # the next read. - tail = self._zip.unconsumed_tail - if tail: - # move the window, make it as large as size demands. For code-clarity, - # we just take the chunk from our map again instead of reusing the unconsumed - # tail. The latter one would safe some memory copying, but we could end up - # with not getting enough data uncompressed, so we had to sort that out as well. - # Now we just assume the worst case, hence the data is uncompressed and the window - # needs to be as large as the uncompressed bytes we want to read. - self._cws = self._cwe - len(tail) - self._cwe = self._cws + size - else: - cws = self._cws - self._cws = self._cwe - self._cwe = cws + size - # END handle tail - - - # if window is too small, make it larger so zip can decompress something - if self._cwe - self._cws < 8: - self._cwe = self._cws + 8 - # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) - - # get the actual window end to be sure we don't use it for computations - self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read - # We feed possibly overlapping chunks, which is why the unconsumed tail - # has to be taken into consideration, as well as the unused data - # if we hit the end of the stream - self._cbr += len(indata) - len(self._zip.unconsumed_tail) - self._br += len(dcompdat) - - if dat: - dcompdat = dat + dcompdat - # END prepend our cached data - - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. - # Recursively resolve that. - # Note: dcompdat can be empty even though we still appear to have bytes - # to read, if we are called by compressed_bytes_read - it manipulates - # us to empty the stream - if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size-len(dcompdat)) - # END handle special case - return dcompdat - - + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, thats all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') + + max_read_size = 512*1024 # currently unused + + def __init__(self, m, close_on_deletion, size=None): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + if size is not None: + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info + self._close = close_on_deletion # close the memmap on deletion ? + + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + + def __del__(self): + if self._close: + self._m.close() + # END handle resource freeing + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + + :return: parsed type_string, size""" + # read header + maxb = 512 # should really be enough, cgit uses 8192 I believe + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find("\0") + type, size = hdr[:hdrend].split(" ") + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 # count terminating \0 + self._buf = StringIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + self._phi = True + + return type, size + + #{ Interface + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + + :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + type, size = inst._parse_header_info() + return type, size, inst + + def data(self): + """:return: random access compatible data we are working on""" + return self._m + + def compressed_bytes_read(self): + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and not self._zip.unused_data: + # manipulate the bytes-read to allow our own read method to coninute + # but keep the window at its current position + self._br = 0 + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr + + #} END interface + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return str() + # END handle depletion + + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = str() + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes durin g + # the next read. + tail = self._zip.unconsumed_tail + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + size + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + size + # END handle tail + + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = buffer(self._m, self._cws, self._cwe - self._cws) + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + dcompdat = self._zip.decompress(indata, size) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + self._cbr += len(indata) - len(self._zip.unconsumed_tail) + self._br += len(dcompdat) + + if dat: + dcompdat = dat + dcompdat + # END prepend our cached data + + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. + # Recursively resolve that. + # Note: dcompdat can be empty even though we still appear to have bytes + # to read, if we are called by compressed_bytes_read - it manipulates + # us to empty the stream + if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: + dcompdat += self.read(size-len(dcompdat)) + # END handle special case + return dcompdat + + class DeltaApplyReader(LazyMixin): - """A reader which dynamically applies pack deltas to a base object, keeping the - memory demands to a minimum. - - The size of the final object is only obtainable once all deltas have been - applied, unless it is retrieved from a pack index. - - The uncompressed Delta has the following layout (MSB being a most significant - bit encoded dynamic size): - - * MSB Source Size - the size of the base against which the delta was created - * MSB Target Size - the size of the resulting data after the delta was applied - * A list of one byte commands (cmd) which are followed by a specific protocol: - - * cmd & 0x80 - copy delta_data[offset:offset+size] - - * Followed by an encoded offset into the delta data - * Followed by an encoded size of the chunk to copy - - * cmd & 0x7f - insert - - * insert cmd bytes from the delta buffer into the output stream - - * cmd == 0 - invalid operation ( or error in delta stream ) - """ - __slots__ = ( - "_bstream", # base stream to which to apply the deltas - "_dstreams", # tuple of delta stream readers - "_mm_target", # memory map of the delta-applied data - "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read - ) - - #{ Configuration - k_max_memory_move = 250*1000*1000 - #} END configuration - - def __init__(self, stream_list): - """Initialize this instance with a list of streams, the first stream being - the delta to apply on top of all following deltas, the last stream being the - base object onto which to apply the deltas""" - assert len(stream_list) > 1, "Need at least one delta and one base stream" - - self._bstream = stream_list[-1] - self._dstreams = tuple(stream_list[:-1]) - self._br = 0 - - def _set_cache_too_slow_without_c(self, attr): - # the direct algorithm is fastest and most direct if there is only one - # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python, every function call costs - # huge amounts of time - # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: - if len(self._dstreams) == 1: - return self._set_cache_brute_(attr) - - # Aggregate all deltas into one delta in reverse order. Hence we take - # the last delta, and reverse-merge its ancestor delta, until we receive - # the final delta data stream. - # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) - dcl = connect_deltas(self._dstreams) - - # call len directly, as the (optional) c version doesn't implement the sequence - # protocol - if dcl.rbound() == 0: - self._size = 0 - self._mm_target = allocate_memory(0) - return - # END handle empty list - - self._size = dcl.rbound() - self._mm_target = allocate_memory(self._size) - - bbuf = allocate_memory(self._bstream.size) - stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - - # APPLY CHUNKS - write = self._mm_target.write - dcl.apply(bbuf, write) - - self._mm_target.seek(0) - - def _set_cache_brute_(self, attr): - """If we are here, we apply the actual deltas""" - - # TODO: There should be a special case if there is only one stream - # Then the default-git algorithm should perform a tad faster, as the - # delta is not peaked into, causing less overhead. - buffer_info_list = list() - max_target_size = 0 - for dstream in self._dstreams: - buf = dstream.read(512) # read the header information + X - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) - max_target_size = max(max_target_size, target_size) - # END for each delta stream - - # sanity check - the first delta to apply should have the same source - # size as our actual base stream - base_size = self._bstream.size - target_size = max_target_size - - # if we have more than 1 delta to apply, we will swap buffers, hence we must - # assure that all buffers we use are large enough to hold all the results - if len(self._dstreams) > 1: - base_size = target_size = max(base_size, max_target_size) - # END adjust buffer sizes - - - # Allocate private memory map big enough to hold the first base buffer - # We need random access to it - bbuf = allocate_memory(base_size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) - - # allocate memory map large enough for the largest (intermediate) target - # We will use it as scratch space for all delta ops. If the final - # target buffer is smaller than our allocated space, we just use parts - # of it upon return. - tbuf = allocate_memory(target_size) - - # for each delta to apply, memory map the decompressed delta and - # work on the op-codes to reconstruct everything. - # For the actual copying, we use a seek and write pattern of buffer - # slices. - final_target_size = None - for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): - # allocate a buffer to hold all delta data - fill in the data for - # fast access. We do this as we know that reading individual bytes - # from our stream would be slower than necessary ( although possible ) - # The dbuf buffer contains commands after the first two MSB sizes, the - # offset specifies the amount of bytes read to get the sizes. - ddata = allocate_memory(dstream.size - offset) - ddata.write(dbuf) - # read the rest from the stream. The size we give is larger than necessary - stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - - ####################################################################### - if 'c_apply_delta' in globals(): - c_apply_delta(bbuf, ddata, tbuf); - else: - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) - ####################################################################### - - # finally, swap out source and target buffers. The target is now the - # base for the next delta to apply - bbuf, tbuf = tbuf, bbuf - bbuf.seek(0) - tbuf.seek(0) - final_target_size = target_size - # END for each delta to apply - - # its already seeked to 0, constrain it to the actual size - # NOTE: in the end of the loop, it swaps buffers, hence our target buffer - # is not tbuf, but bbuf ! - self._mm_target = bbuf - self._size = final_target_size - - - #{ Configuration - if not has_perf_mod: - _set_cache_ = _set_cache_brute_ - else: - _set_cache_ = _set_cache_too_slow_without_c - - #} END configuration - - def read(self, count=0): - bl = self._size - self._br # bytes left - if count < 1 or count > bl: - count = bl - # NOTE: we could check for certain size limits, and possibly - # return buffers instead of strings to prevent byte copying - data = self._mm_target.read(count) - self._br += len(data) - return data - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Allows to reset the stream to restart reading - - :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - self._br = 0 - self._mm_target.seek(0) - - #{ Interface - - @classmethod - def new(cls, stream_list): - """ - Convert the given list of streams into a stream which resolves deltas - when reading from it. - - :param stream_list: two or more stream objects, first stream is a Delta - to the object that you want to resolve, followed by N additional delta - streams. The list's last stream must be a non-delta stream. - - :return: Non-Delta OPackStream object whose stream can be used to obtain - the decompressed resolved data - :raise ValueError: if the stream list cannot be handled""" - if len(stream_list) < 2: - raise ValueError("Need at least two streams") - # END single object special handling - - if stream_list[-1].type_id in delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) - # END check stream - - return cls(stream_list) - - #} END interface - - - #{ OInfo like Interface - - @property - def type(self): - return self._bstream.type - - @property - def type_id(self): - return self._bstream.type_id - - @property - def size(self): - """:return: number of uncompressed bytes in the stream""" - return self._size - - #} END oinfo like interface - - + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) + + #{ Configuration + k_max_memory_move = 250*1000*1000 + #} END configuration + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 + + def _set_cache_too_slow_without_c(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) + dcl = connect_deltas(self._dstreams) + + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.rbound() == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl.rbound() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) + + # APPLY CHUNKS + write = self._mm_target.write + dcl.apply(bbuf, write) + + self._mm_target.seek(0) + + def _set_cache_brute_(self, attr): + """If we are here, we apply the actual deltas""" + + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. + buffer_info_list = list() + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + + + # Allocate private memory map big enough to hold the first base buffer + # We need random access to it + bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it upon return. + tbuf = allocate_memory(target_size) + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + + ####################################################################### + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf); + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + ####################################################################### + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + final_target_size = target_size + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size + + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c + + #} END configuration + + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self._br = 0 + self._mm_target.seek(0) + + #{ Interface + + @classmethod + def new(cls, stream_list): + """ + Convert the given list of streams into a stream which resolves deltas + when reading from it. + + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + return cls(stream_list) + + #} END interface + + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + #} END RO streams #{ W Streams class Sha1Writer(object): - """Simple stream writer which produces a sha whenever you like as it degests - everything it is supposed to write""" - __slots__ = "sha1" - - def __init__(self): - self.sha1 = make_sha() - - #{ Stream Interface - - def write(self, data): - """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" - self.sha1.update(data) - return len(data) - - # END stream interface - - #{ Interface - - def sha(self, as_hex = False): - """:return: sha so far - :param as_hex: if True, sha will be hex-encoded, binary otherwise""" - if as_hex: - return self.sha1.hexdigest() - return self.sha1.digest() - - #} END interface + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha() + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex = False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface class FlexibleSha1Writer(Sha1Writer): - """Writer producing a sha1 while passing on the written bytes to the given - write function""" - __slots__ = 'writer' - - def __init__(self, writer): - Sha1Writer.__init__(self) - self.writer = writer - - def write(self, data): - Sha1Writer.write(self, data) - self.writer(data) + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) class ZippedStoreShaWriter(Sha1Writer): - """Remembers everything someone writes to it and generates a sha""" - __slots__ = ('buf', 'zip') - def __init__(self): - Sha1Writer.__init__(self) - self.buf = StringIO() - self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - - def __getattr__(self, attr): - return getattr(self.buf, attr) - - def write(self, data): - alen = Sha1Writer.write(self, data) - self.buf.write(self.zip.compress(data)) - return alen - - def close(self): - self.buf.write(self.zip.flush()) - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Seeking currently only supports to rewind written data - Multiple writes are not supported""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - self.buf.seek(0) - - def getvalue(self): - """:return: string value from the current stream position to the end""" - return self.buf.getvalue() + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() class FDCompressedSha1Writer(Sha1Writer): - """Digests data written to it, making the sha available, then compress the - data and write it to the file descriptor - - **Note:** operates on raw file descriptors - **Note:** for this to work, you have to use the close-method of this instance""" - __slots__ = ("fd", "sha1", "zip") - - # default exception - exc = IOError("Failed to write all bytes to filedescriptor") - - def __init__(self, fd): - super(FDCompressedSha1Writer, self).__init__() - self.fd = fd - self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - - #{ Stream Interface - - def write(self, data): - """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" - self.sha1.update(data) - cdata = self.zip.compress(data) - bytes_written = write(self.fd, cdata) - if bytes_written != len(cdata): - raise self.exc - return len(data) - - def close(self): - remainder = self.zip.flush() - if write(self.fd, remainder) != len(remainder): - raise self.exc - return close(self.fd) - - #} END stream interface + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super(FDCompressedSha1Writer, self).__init__() + self.fd = fd + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): + raise self.exc + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface class FDStream(object): - """A simple wrapper providing the most basic functions on a file descriptor - with the fileobject interface. Cannot use os.fdopen as the resulting stream - takes ownership""" - __slots__ = ("_fd", '_pos') - def __init__(self, fd): - self._fd = fd - self._pos = 0 - - def write(self, data): - self._pos += len(data) - os.write(self._fd, data) - - def read(self, count=0): - if count == 0: - count = os.path.getsize(self._filepath) - # END handle read everything - - bytes = os.read(self._fd, count) - self._pos += len(bytes) - return bytes - - def fileno(self): - return self._fd - - def tell(self): - return self._pos - - def close(self): - close(self._fd) + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + def close(self): + close(self._fd) class NullStream(object): - """A stream that does nothing but providing a stream interface. - Use it like /dev/null""" - __slots__ = tuple() - - def read(self, size=0): - return '' - - def close(self): - pass - - def write(self, data): - return len(data) + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) #} END W streams diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 760f531be..f8059447f 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -7,10 +7,10 @@ #{ Initialization def _init_pool(): - """Assure the pool is actually threaded""" - size = 2 - print "Setting ThreadPool to %i" % size - gitdb.util.pool.set_size(size) + """Assure the pool is actually threaded""" + size = 2 + print "Setting ThreadPool to %i" % size + gitdb.util.pool.set_size(size) #} END initialization diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 4af4483c7..62614ee5c 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -4,21 +4,21 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Base classes for object db testing""" from gitdb.test.lib import ( - with_rw_directory, - with_packs_rw, - ZippedStoreShaWriter, - fixture_path, - TestBase - ) + with_rw_directory, + with_packs_rw, + ZippedStoreShaWriter, + fixture_path, + TestBase + ) from gitdb.stream import Sha1Writer from gitdb.base import ( - IStream, - OStream, - OInfo - ) - + IStream, + OStream, + OInfo + ) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type @@ -28,181 +28,181 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') - + class TestDBBase(TestBase): - """Base class providing testing routines on databases""" - - # data - two_lines = "1234\nhello world" - all_data = (two_lines, ) - - def _assert_object_writing_simple(self, db): - # write a bunch of objects and query their streams and info - null_objs = db.size() - ni = 250 - for i in xrange(ni): - data = pack(">L", i) - istream = IStream(str_blob_type, len(data), StringIO(data)) - new_istream = db.store(istream) - assert new_istream is istream - assert db.has_object(istream.binsha) - - info = db.info(istream.binsha) - assert isinstance(info, OInfo) - assert info.type == istream.type and info.size == istream.size - - stream = db.stream(istream.binsha) - assert isinstance(stream, OStream) - assert stream.binsha == info.binsha and stream.type == info.type - assert stream.read() == data - # END for each item - - assert db.size() == null_objs + ni - shas = list(db.sha_iter()) - assert len(shas) == db.size() - assert len(shas[0]) == 20 - - - def _assert_object_writing(self, db): - """General tests to verify object writing, compatible to ObjectDBW - **Note:** requires write access to the database""" - # start in 'dry-run' mode, using a simple sha1 writer - ostreams = (ZippedStoreShaWriter, None) - for ostreamcls in ostreams: - for data in self.all_data: - dry_run = ostreamcls is not None - ostream = None - if ostreamcls is not None: - ostream = ostreamcls() - assert isinstance(ostream, Sha1Writer) - # END create ostream - - prev_ostream = db.set_ostream(ostream) - assert type(prev_ostream) in ostreams or prev_ostream in ostreams - - istream = IStream(str_blob_type, len(data), StringIO(data)) - - # store returns same istream instance, with new sha set - my_istream = db.store(istream) - sha = istream.binsha - assert my_istream is istream - assert db.has_object(sha) != dry_run - assert len(sha) == 20 - - # verify data - the slow way, we want to run code - if not dry_run: - info = db.info(sha) - assert str_blob_type == info.type - assert info.size == len(data) - - ostream = db.stream(sha) - assert ostream.read() == data - assert ostream.type == str_blob_type - assert ostream.size == len(data) - else: - self.failUnlessRaises(BadObject, db.info, sha) - self.failUnlessRaises(BadObject, db.stream, sha) - - # DIRECT STREAM COPY - # our data hase been written in object format to the StringIO - # we pasesd as output stream. No physical database representation - # was created. - # Test direct stream copy of object streams, the result must be - # identical to what we fed in - ostream.seek(0) - istream.stream = ostream - assert istream.binsha is not None - prev_sha = istream.binsha - - db.set_ostream(ZippedStoreShaWriter()) - db.store(istream) - assert istream.binsha == prev_sha - new_ostream = db.ostream() - - # note: only works as long our store write uses the same compression - # level, which is zip_best - assert ostream.getvalue() == new_ostream.getvalue() - # END for each data set - # END for each dry_run mode - - def _assert_object_writing_async(self, db): - """Test generic object writing using asynchronous access""" - ni = 5000 - def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - data = str(data_src + offset) - yield IStream(str_blob_type, len(data), StringIO(data)) - # END for each item - # END generator utility - - # for now, we are very trusty here as we expect it to work if it worked - # in the single-stream case - - # write objects - reader = IteratorReader(istream_generator()) - istream_reader = db.store_async(reader) - istreams = istream_reader.read() # read all - assert istream_reader.task().error() is None - assert len(istreams) == ni - - for stream in istreams: - assert stream.error is None - assert len(stream.binsha) == 20 - assert isinstance(stream, IStream) - # END assert each stream - - # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.binsha for istream in istreams ) - hasobject_reader = db.has_object_async(reader) - count = 0 - for sha, has_object in hasobject_reader: - assert has_object - count += 1 - # END for each sha - assert count == ni - - # read the objects we have just written - reader = IteratorReader( istream.binsha for istream in istreams ) - ostream_reader = db.stream_async(reader) - - # read items individually to prevent hitting possible sys-limits - count = 0 - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert ostream_reader.task().error() is None - assert count == ni - - # get info about our items - reader = IteratorReader( istream.binsha for istream in istreams ) - info_reader = db.info_async(reader) - - count = 0 - for oinfo in info_reader: - assert isinstance(oinfo, OInfo) - count += 1 - # END for each oinfo instance - assert count == ni - - - # combined read-write using a converter - # add 2500 items, and obtain their output streams - nni = 2500 - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - - istream_reader = db.store_async(reader) - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = db.stream_async(istream_reader) - - count = 0 - # read it individually, otherwise we might run into the ulimit - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert count == nni - - + """Base class providing testing routines on databases""" + + # data + two_lines = "1234\nhello world" + all_data = (two_lines, ) + + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in xrange(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), StringIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.binsha) + + info = db.info(istream.binsha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.binsha) + assert isinstance(stream, OStream) + assert stream.binsha == info.binsha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + **Note:** requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + + istream = IStream(str_blob_type, len(data), StringIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.binsha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 20 + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert str_blob_type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == str_blob_type + assert ostream.size == len(data) + else: + self.failUnlessRaises(BadObject, db.info, sha) + self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.binsha is not None + prev_sha = istream.binsha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.binsha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip_best + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode + + def _assert_object_writing_async(self, db): + """Test generic object writing using asynchronous access""" + ni = 5000 + def istream_generator(offset=0, ni=ni): + for data_src in xrange(ni): + data = str(data_src + offset) + yield IStream(str_blob_type, len(data), StringIO(data)) + # END for each item + # END generator utility + + # for now, we are very trusty here as we expect it to work if it worked + # in the single-stream case + + # write objects + reader = IteratorReader(istream_generator()) + istream_reader = db.store_async(reader) + istreams = istream_reader.read() # read all + assert istream_reader.task().error() is None + assert len(istreams) == ni + + for stream in istreams: + assert stream.error is None + assert len(stream.binsha) == 20 + assert isinstance(stream, IStream) + # END assert each stream + + # test has-object-async - we must have all previously added ones + reader = IteratorReader( istream.binsha for istream in istreams ) + hasobject_reader = db.has_object_async(reader) + count = 0 + for sha, has_object in hasobject_reader: + assert has_object + count += 1 + # END for each sha + assert count == ni + + # read the objects we have just written + reader = IteratorReader( istream.binsha for istream in istreams ) + ostream_reader = db.stream_async(reader) + + # read items individually to prevent hitting possible sys-limits + count = 0 + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert ostream_reader.task().error() is None + assert count == ni + + # get info about our items + reader = IteratorReader( istream.binsha for istream in istreams ) + info_reader = db.info_async(reader) + + count = 0 + for oinfo in info_reader: + assert isinstance(oinfo, OInfo) + count += 1 + # END for each oinfo instance + assert count == ni + + + # combined read-write using a converter + # add 2500 items, and obtain their output streams + nni = 2500 + reader = IteratorReader(istream_generator(offset=ni, ni=nni)) + istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] + + istream_reader = db.store_async(reader) + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = db.stream_async(istream_reader) + + count = 0 + # read it individually, otherwise we might run into the ulimit + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert count == nni + + diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 310116351..1ef577aa3 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,41 +7,41 @@ from gitdb.db import GitDB from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex - + class TestGitDB(TestDBBase): - - def test_reading(self): - gdb = GitDB(fixture_path('../../../.git/objects')) - - # we have packs and loose objects, alternates doesn't necessarily exist - assert 1 < len(gdb.databases()) < 4 - - # access should be possible - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") - assert isinstance(gdb.info(gitdb_sha), OInfo) - assert isinstance(gdb.stream(gitdb_sha), OStream) - assert gdb.size() > 200 - sha_list = list(gdb.sha_iter()) - assert len(sha_list) == gdb.size() - - - # This is actually a test for compound functionality, but it doesn't - # have a separate test module - # test partial shas - # this one as uneven and quite short - assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") - - # mix even/uneven hexshas - for i, binsha in enumerate(sha_list): - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha - # END for each sha - - self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") - - @with_rw_directory - def test_writing(self, path): - gdb = GitDB(path) - - # its possible to write objects - self._assert_object_writing(gdb) - self._assert_object_writing_async(gdb) + + def test_reading(self): + gdb = GitDB(fixture_path('../../../.git/objects')) + + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + assert gdb.size() > 200 + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module + # test partial shas + # this one as uneven and quite short + assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha + # END for each sha + + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) + self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index ee2d78d08..d7e1d01b0 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -6,29 +6,29 @@ from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex - + class TestLooseDB(TestDBBase): - - @with_rw_directory - def test_basics(self, path): - ldb = LooseObjectDB(path) - - # write data - self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) - - # verify sha iteration and size - shas = list(ldb.sha_iter()) - assert shas and len(shas[0]) == 20 - - assert len(shas) == ldb.size() - - # verify find short object - long_sha = bin_to_hex(shas[-1]) - for short_sha in (long_sha[:20], long_sha[:5]): - assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha - # END for each sha - - self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') - # raises if no object could be foudn - + + @with_rw_directory + def test_basics(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) + + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be foudn + diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 188cb0a93..df428e2b7 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -4,27 +4,27 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( - MemoryDB, - LooseObjectDB - ) - + MemoryDB, + LooseObjectDB + ) + class TestMemoryDB(TestDBBase): - - @with_rw_directory - def test_writing(self, path): - mdb = MemoryDB() - - # write data - self._assert_object_writing_simple(mdb) - - # test stream copy - ldb = LooseObjectDB(path) - assert ldb.size() == 0 - num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) - assert num_streams_copied == mdb.size() - - assert ldb.size() == mdb.size() - for sha in mdb.sha_iter(): - assert ldb.has_object(sha) - assert ldb.stream(sha).read() == mdb.stream(sha).read() - # END verify objects where copied and are equal + + @with_rw_directory + def test_writing(self, path): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index e8ba6f8fc..f4cb5bbc6 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -12,62 +12,62 @@ import random class TestPackDB(TestDBBase): - - @with_rw_directory - @with_packs_rw - def test_writing(self, path): - pdb = PackedDB(path) - - # on demand, we init our pack cache - num_packs = len(pdb.entities()) - assert pdb._st_mtime != 0 - - # test pack directory changed: - # packs removed - rename a file, should affect the glob - pack_path = pdb.entities()[0].pack().path() - new_pack_path = pack_path + "renamed" - os.rename(pack_path, new_pack_path) - - pdb.update_cache(force=True) - assert len(pdb.entities()) == num_packs - 1 - - # packs added - os.rename(new_pack_path, pack_path) - pdb.update_cache(force=True) - assert len(pdb.entities()) == num_packs - - # bang on the cache - # access the Entities directly, as there is no iteration interface - # yet ( or required for now ) - sha_list = list(pdb.sha_iter()) - assert len(sha_list) == pdb.size() - - # hit all packs in random order - random.shuffle(sha_list) - - for sha in sha_list: - info = pdb.info(sha) - stream = pdb.stream(sha) - # END for each sha to query - - - # test short finding - be a bit more brutal here - max_bytes = 19 - min_bytes = 2 - num_ambiguous = 0 - for i, sha in enumerate(sha_list): - short_sha = sha[:max((i % max_bytes), min_bytes)] - try: - assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha - except AmbiguousObjectName: - num_ambiguous += 1 - pass # valid, we can have short objects - # END exception handling - # END for each sha to find - - # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... - # assert num_ambiguous - - # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) + + @with_rw_directory + @with_packs_rw + def test_writing(self, path): + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = len(pdb.entities()) + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb.entities()[0].pack().path() + new_pack_path = pack_path + "renamed" + os.rename(pack_path, new_pack_path) + + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs + + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + info = pdb.info(sha) + stream = pdb.stream(sha) + # END for each sha to query + + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambigious ... + # assert num_ambiguous + + # non-existing + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 0d8eeebb3..1637bff74 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -6,55 +6,55 @@ from gitdb.db import ReferenceDB from gitdb.util import ( - NULL_BIN_SHA, - hex_to_bin - ) + NULL_BIN_SHA, + hex_to_bin + ) import os - + class TestReferenceDB(TestDBBase): - - def make_alt_file(self, alt_path, alt_list): - """Create an alternates file which contains the given alternates. - The list can be empty""" - alt_file = open(alt_path, "wb") - for alt in alt_list: - alt_file.write(alt + "\n") - alt_file.close() - - @with_rw_directory - def test_writing(self, path): - NULL_BIN_SHA = '\0' * 20 - - alt_path = os.path.join(path, 'alternates') - rdb = ReferenceDB(alt_path) - assert len(rdb.databases()) == 0 - assert rdb.size() == 0 - assert len(list(rdb.sha_iter())) == 0 - - # try empty, non-existing - assert not rdb.has_object(NULL_BIN_SHA) - - - # setup alternate file - # add two, one is invalid - own_repo_path = fixture_path('../../../.git/objects') # use own repo - self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) - rdb.update_cache() - assert len(rdb.databases()) == 1 - - # we should now find a default revision of ours - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") - assert rdb.has_object(gitdb_sha) - - # remove valid - self.make_alt_file(alt_path, ["just/one/invalid/path"]) - rdb.update_cache() - assert len(rdb.databases()) == 0 - - # add valid - self.make_alt_file(alt_path, [own_repo_path]) - rdb.update_cache() - assert len(rdb.databases()) == 1 - - + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + alt_file = open(alt_path, "wb") + for alt in alt_list: + alt_file.write(alt + "\n") + alt_file.close() + + @with_rw_directory + def test_writing(self, path): + NULL_BIN_SHA = '\0' * 20 + + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 + + # try empty, non-existing + assert not rdb.has_object(NULL_BIN_SHA) + + + # setup alternate file + # add two, one is invalid + own_repo_path = fixture_path('../../../.git/objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + assert rdb.has_object(gitdb_sha) + + # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 + + # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 50645be65..ac8473a4e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,12 +4,12 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( - OStream, - ) + OStream, + ) from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter - ) + Sha1Writer, + ZippedStoreShaWriter + ) from gitdb.util import zlib @@ -29,134 +29,134 @@ #{ Bases class TestBase(unittest.TestCase): - """Base class for all tests""" - + """Base class for all tests""" + #} END bases #{ Decorators def with_rw_directory(func): - """Create a temporary directory which can be written to, remove it if the - test suceeds, but leave it otherwise to aid additional debugging""" - def wrapper(self): - path = tempfile.mktemp(prefix=func.__name__) - os.mkdir(path) - keep = False - try: - try: - return func(self, path) - except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) - keep = True - raise - finally: - # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as - # memory maps closed, once objects go out of scope. For some reason - # though this is not the case here unless we collect explicitly. - if not keep: - gc.collect() - shutil.rmtree(path) - # END handle exception - # END wrapper - - wrapper.__name__ = func.__name__ - return wrapper + """Create a temporary directory which can be written to, remove it if the + test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): + path = tempfile.mktemp(prefix=func.__name__) + os.mkdir(path) + keep = False + try: + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + keep = True + raise + finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + if not keep: + gc.collect() + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper def with_packs_rw(func): - """Function that provides a path into which the packs for testing should be - copied. Will pass on the path to the actual function afterwards""" - def wrapper(self, path): - src_pack_glob = fixture_path('packs/*') - copy_files_globbed(src_pack_glob, path, hard_link_ok=True) - return func(self, path) - # END wrapper - - wrapper.__name__ = func.__name__ - return wrapper + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper #} END decorators #{ Routines def fixture_path(relapath=''): - """:return: absolute path into the fixture directory - :param relapath: relative path into the fixtures directory, or '' - to obtain the fixture directory itself""" - return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) - + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): - """Copy all files found according to the given source glob into the target directory - :param hard_link_ok: if True, hard links will be created if possible. Otherwise - the files will be copied""" - for src_file in glob.glob(source_glob): - if hard_link_ok and hasattr(os, 'link'): - target = os.path.join(target_dir, os.path.basename(src_file)) - try: - os.link(src_file, target) - except OSError: - shutil.copy(src_file, target_dir) - # END handle cross device links ( and resulting failure ) - else: - shutil.copy(src_file, target_dir) - # END try hard link - # END for each file to copy - + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + def make_bytes(size_in_bytes, randomize=False): - """:return: string with given size in bytes - :param randomize: try to produce a very random stream""" - actual_size = size_in_bytes / 4 - producer = xrange(actual_size) - if randomize: - producer = list(producer) - random.shuffle(producer) - # END randomize - a = array('i', producer) - return a.tostring() + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tostring() def make_object(type, data): - """:return: bytes resembling an uncompressed object""" - odata = "blob %i\0" % len(data) - return odata + data - + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata + data + def make_memory_file(size_in_bytes, randomize=False): - """:return: tuple(size_of_stream, stream) - :param randomize: try to produce a very random stream""" - d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) #} END routines #{ Stream Utilities class DummyStream(object): - def __init__(self): - self.was_read = False - self.bytes = 0 - self.closed = False - - def read(self, size): - self.was_read = True - self.bytes = size - - def close(self): - self.closed = True - - def _assert(self): - assert self.was_read + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read class DeriveTest(OStream): - def __init__(self, sha, type, size, stream, *args, **kwargs): - self.myarg = kwargs.pop('myarg') - self.args = args - - def _assert(self): - assert self.args - assert self.myarg + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg #} END stream utilitiess diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 761113d51..3563fcfbe 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -16,12 +16,12 @@ #{ Utilities def resolve_or_fail(env_var): - """:return: resolved environment variable or raise EnvironmentError""" - try: - return os.environ[env_var] - except KeyError: - raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) - # END exception handling + """:return: resolved environment variable or raise EnvironmentError""" + try: + return os.environ[env_var] + except KeyError: + raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) + # END exception handling #} END utilities @@ -29,26 +29,26 @@ def resolve_or_fail(env_var): #{ Base Classes class TestBigRepoR(TestBase): - """TestCase providing access to readonly 'big' repositories using the following - member variables: - - * gitrepopath - - * read-only base path of the git source repository, i.e. .../git/.git""" - - #{ Invariants - head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' - head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' - #} END invariants - - @classmethod - def setUpAll(cls): - try: - super(TestBigRepoR, cls).setUpAll() - except AttributeError: - pass - cls.gitrepopath = resolve_or_fail(k_env_git_repo) - assert cls.gitrepopath.endswith('.git') - - + """TestCase providing access to readonly 'big' repositories using the following + member variables: + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git""" + + #{ Invariants + head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' + head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' + #} END invariants + + @classmethod + def setUpAll(cls): + try: + super(TestBigRepoR, cls).setUpAll() + except AttributeError: + pass + cls.gitrepopath = resolve_or_fail(k_env_git_repo) + assert cls.gitrepopath.endswith('.git') + + #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 20618024d..63856e218 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -4,8 +4,8 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" from lib import ( - TestBigRepoR - ) + TestBigRepoR + ) from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB @@ -18,76 +18,76 @@ from nose import SkipTest class TestPackedDBPerformance(TestBigRepoR): - - def test_pack_random_access(self): - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - # sha lookup - st = time() - sha_list = list(pdb.sha_iter()) - elapsed = time() - st - ns = len(sha_list) - print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) - - # sha lookup: best-case and worst case access - pdb_pack_info = pdb._pack_info - # END shuffle shas - st = time() - for sha in sha_list: - pdb_pack_info(sha) - # END for each sha to look up - elapsed = time() - st - - # discard cache - del(pdb._entities) - pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) - # END for each random mode - - # query info and streams only - max_items = 10000 # can wait longer when testing memory - for pdb_fun in (pdb.info, pdb.stream): - st = time() - for sha in sha_list[:max_items]: - pdb_fun(sha) - elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) - # END for each function - - # retrieve stream and read all - max_items = 5000 - pdb_stream = pdb.stream - total_size = 0 - st = time() - for sha in sha_list[:max_items]: - stream = pdb_stream(sha) - stream.read() - total_size += stream.size - elapsed = time() - st - total_kib = total_size / 1000 - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - - def test_correctness(self): - raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - # disabled for now as it used to work perfectly, checking big repositories takes a long time - print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" - for crc in range(2): - count = 0 - st = time() - for entity in pdb.entities(): - pack_verify = entity.is_valid_stream - sha_by_index = entity.index().sha - for index in xrange(entity.index().size()): - try: - assert pack_verify(sha_by_index(index), use_crc=crc) - count += 1 - except UnsupportedOperation: - pass - # END ignore old indices - # END for each index - # END for each entity - elapsed = time() - st - print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) - # END for each verify mode - + + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + + # discard cache + del(pdb._entities) + pdb.entities() + print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + # END for each random mode + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + # END for each function + + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + + def test_correctness(self): + raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in xrange(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 + except UnsupportedOperation: + pass + # END ignore old indices + # END for each index + # END for each entity + elapsed = time() - st + print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + # END for each verify mode + diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 3c40ed0fb..c66e60cba 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -4,8 +4,8 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" from lib import ( - TestBigRepoR - ) + TestBigRepoR + ) from gitdb.db.pack import PackedDB from gitdb.stream import NullStream @@ -17,63 +17,63 @@ from nose import SkipTest class CountedNullStream(NullStream): - __slots__ = '_bw' - def __init__(self): - self._bw = 0 - - def bytes_written(self): - return self._bw - - def write(self, d): - self._bw += NullStream.write(self, d) - + __slots__ = '_bw' + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + class TestPackStreamingPerformance(TestBigRepoR): - - def test_pack_writing(self): - # see how fast we can write a pack from object streams. - # This will not be fast, as we take time for decompressing the streams as well - ostream = CountedNullStream() - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - ni = 5000 - count = 0 - total_size = 0 - st = time() - for sha in pdb.sha_iter(): - count += 1 - pdb.stream(sha) - if count == ni: - break - #END gather objects for pack-writing - elapsed = time() - st - print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) - - st = time() - PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) - elapsed = time() - st - total_kb = ostream.bytes_written() / 1000 - print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) - - - def test_stream_reading(self): - raise SkipTest() - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - # streaming only, meant for --with-profile runs - ni = 5000 - count = 0 - pdb_stream = pdb.stream - total_size = 0 - st = time() - for sha in pdb.sha_iter(): - if count == ni: - break - stream = pdb_stream(sha) - stream.read() - total_size += stream.size - count += 1 - elapsed = time() - st - total_kib = total_size / 1000 - print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) - + + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 5000 + count = 0 + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + count += 1 + pdb.stream(sha) + if count == ni: + break + #END gather objects for pack-writing + elapsed = time() - st + print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + + st = time() + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + + + def test_stream_reading(self): + raise SkipTest() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index f5f2e2e4d..010003d4b 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -8,16 +8,16 @@ from gitdb.base import * from gitdb.stream import * from gitdb.util import ( - pool, - bin_to_hex - ) + pool, + bin_to_hex + ) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size from async import ( - IteratorReader, - ChannelThreadTask, - ) + IteratorReader, + ChannelThreadTask, + ) from cStringIO import StringIO from time import time @@ -28,168 +28,168 @@ from lib import ( - TestBigRepoR, - make_memory_file, - with_rw_directory - ) + TestBigRepoR, + make_memory_file, + with_rw_directory + ) #{ Utilities def read_chunked_stream(stream): - total = 0 - while True: - chunk = stream.read(chunk_size) - total += len(chunk) - if len(chunk) < chunk_size: - break - # END read stream loop - assert total == stream.size - return stream - - + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + class TestStreamReader(ChannelThreadTask): - """Expects input streams and reads them in chunks. It will read one at a time, - requireing a queue chunk of size 1""" - def __init__(self, *args): - super(TestStreamReader, self).__init__(*args) - self.fun = read_chunked_stream - self.max_chunksize = 1 - + """Expects input streams and reads them in chunks. It will read one at a time, + requireing a queue chunk of size 1""" + def __init__(self, *args): + super(TestStreamReader, self).__init__(*args) + self.fun = read_chunked_stream + self.max_chunksize = 1 + #} END utilities class TestObjDBPerformance(TestBigRepoR): - - large_data_size_bytes = 1000*1000*50 # some MiB should do it - moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - - @with_rw_directory - def test_large_data_streaming(self, path): - ldb = LooseObjectDB(path) - string_ios = list() # list of streams we previously created - - # serial mode - for randomize in range(2): - desc = (randomize and 'random ') or '' - print >> sys.stderr, "Creating %s data ..." % desc - st = time() - size, stream = make_memory_file(self.large_data_size_bytes, randomize) - elapsed = time() - st - print >> sys.stderr, "Done (in %f s)" % elapsed - string_ios.append(stream) - - # writing - due to the compression it will seem faster than it is - st = time() - sha = ldb.store(IStream('blob', size, stream)).binsha - elapsed_add = time() - st - assert ldb.has_object(sha) - db_file = ldb.readable_db_object_path(bin_to_hex(sha)) - fsize_kib = os.path.getsize(db_file) / 1000 - - - size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) - - # reading all at once - st = time() - ostream = ldb.stream(sha) - shadata = ostream.read() - elapsed_readall = time() - st - - stream.seek(0) - assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) - - - # reading in chunks of 1 MiB - cs = 512*1000 - chunks = list() - st = time() - ostream = ldb.stream(sha) - while True: - data = ostream.read(cs) - chunks.append(data) - if len(data) < cs: - break - # END read in chunks - elapsed_readchunks = time() - st - - stream.seek(0) - assert ''.join(chunks) == stream.getvalue() - - cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) - - # del db file so we keep something to do - os.remove(db_file) - # END for each randomization factor - - - # multi-threaded mode - # want two, should be supported by most of todays cpus - pool.set_size(2) - total_kib = 0 - nsios = len(string_ios) - for stream in string_ios: - stream.seek(0) - total_kib += len(stream.getvalue()) / 1000 - # END rewind - - def istream_iter(): - for stream in string_ios: - stream.seek(0) - yield IStream(str_blob_type, len(stream.getvalue()), stream) - # END for each stream - # END util - - # write multiple objects at once, involving concurrent compression - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - st = time() - istreams = istream_reader.read(nsios) - assert len(istreams) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # decompress multiple at once, by reading them - # chunk size is not important as the stream will not really be decompressed - - # until its read - istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.task().max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # store the files, and read them back. For the reading, we use a task - # as well which is chunked into one item per task. Reading all will - # very quickly result in two threads handling two bytestreams of - # chained compression/decompression streams - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - istream_to_sha = lambda items: [ i.binsha for i in items ] - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + large_data_size_bytes = 1000*1000*50 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + + # serial mode + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + string_ios.append(stream) + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).binsha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) + fsize_kib = os.path.getsize(db_file) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so we keep something to do + os.remove(db_file) + # END for each randomization factor + + + # multi-threaded mode + # want two, should be supported by most of todays cpus + pool.set_size(2) + total_kib = 0 + nsios = len(string_ios) + for stream in string_ios: + stream.seek(0) + total_kib += len(stream.getvalue()) / 1000 + # END rewind + + def istream_iter(): + for stream in string_ios: + stream.seek(0) + yield IStream(str_blob_type, len(stream.getvalue()), stream) + # END for each stream + # END util + + # write multiple objects at once, involving concurrent compression + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + st = time() + istreams = istream_reader.read(nsios) + assert len(istreams) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # decompress multiple at once, by reading them + # chunk size is not important as the stream will not really be decompressed + + # until its read + istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + output_reader.task().max_chunksize = 1 + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # store the files, and read them back. For the reading, we use a task + # as well which is chunked into one item per task. Reading all will + # very quickly result in two threads handling two bytestreams of + # chained compression/decompression streams + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + istream_to_sha = lambda items: [ i.binsha for i in items ] + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + output_reader.max_chunksize = 1 + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 1b20faf87..d4ce428c3 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -4,95 +4,95 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( - TestBase, - DummyStream, - DeriveTest, - ) + TestBase, + DummyStream, + DeriveTest, + ) from gitdb import * from gitdb.util import ( - NULL_BIN_SHA - ) + NULL_BIN_SHA + ) from gitdb.typ import ( - str_blob_type - ) + str_blob_type + ) class TestBaseTypes(TestBase): - - def test_streams(self): - # test info - sha = NULL_BIN_SHA - s = 20 - blob_id = 3 - - info = OInfo(sha, str_blob_type, s) - assert info.binsha == sha - assert info.type == str_blob_type - assert info.type_id == blob_id - assert info.size == s - - # test pack info - # provides type_id - pinfo = OPackInfo(0, blob_id, s) - assert pinfo.type == str_blob_type - assert pinfo.type_id == blob_id - assert pinfo.pack_offset == 0 - - dpinfo = ODeltaPackInfo(0, blob_id, s, sha) - assert dpinfo.type == str_blob_type - assert dpinfo.type_id == blob_id - assert dpinfo.delta_info == sha - assert dpinfo.pack_offset == 0 - - - # test ostream - stream = DummyStream() - ostream = OStream(*(info + (stream, ))) - assert ostream.stream is stream - ostream.read(15) - stream._assert() - assert stream.bytes == 15 - ostream.read(20) - assert stream.bytes == 20 - - # test packstream - postream = OPackStream(*(pinfo + (stream, ))) - assert postream.stream is stream - postream.read(10) - stream._assert() - assert stream.bytes == 10 - - # test deltapackstream - dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream - dpostream.read(5) - stream._assert() - assert stream.bytes == 5 - - # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - - # test istream - istream = IStream(str_blob_type, s, stream) - assert istream.binsha == None - istream.binsha = sha - assert istream.binsha == sha - - assert len(istream.binsha) == 20 - assert len(istream.hexsha) == 40 - - assert istream.size == s - istream.size = s * 2 - istream.size == s * 2 - assert istream.type == str_blob_type - istream.type = "something" - assert istream.type == "something" - assert istream.stream is stream - istream.stream = None - assert istream.stream is None - - assert istream.error is None - istream.error = Exception() - assert isinstance(istream.error, Exception) + + def test_streams(self): + # test info + sha = NULL_BIN_SHA + s = 20 + blob_id = 3 + + info = OInfo(sha, str_blob_type, s) + assert info.binsha == sha + assert info.type == str_blob_type + assert info.type_id == blob_id + assert info.size == s + + # test pack info + # provides type_id + pinfo = OPackInfo(0, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 753177560..611ae4299 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -7,58 +7,58 @@ from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool - + from cStringIO import StringIO from async import IteratorReader - + class TestExamples(TestBase): - - def test_base(self): - ldb = LooseObjectDB(fixture_path("../../../.git/objects")) - - for sha1 in ldb.sha_iter(): - oinfo = ldb.info(sha1) - ostream = ldb.stream(sha1) - assert oinfo[:3] == ostream[:3] - - assert len(ostream.read()) == ostream.size - assert ldb.has_object(oinfo.binsha) - # END for each sha in database - # assure we close all files - try: - del(ostream) - del(oinfo) - except UnboundLocalError: - pass - # END ignore exception if there are no loose objects - - data = "my data" - istream = IStream("blob", len(data), StringIO(data)) - - # the object does not yet have a sha - assert istream.binsha is None - ldb.store(istream) - # now the sha is set - assert len(istream.binsha) == 20 - assert ldb.has_object(istream.binsha) - - - # async operation - # Create a reader from an iterator - reader = IteratorReader(ldb.sha_iter()) - - # get reader for object streams - info_reader = ldb.stream_async(reader) - - # read one - info = info_reader.read(1)[0] - - # read all the rest until depletion - ostreams = info_reader.read() - - # set the pool to use two threads - pool.set_size(2) - - # synchronize the mode of operation - pool.set_size(0) + + def test_base(self): + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + # assure we close all files + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + + + # async operation + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 4a7f1caf2..779155a2a 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -4,23 +4,23 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" from lib import ( - TestBase, - with_rw_directory, - with_packs_rw, - fixture_path - ) + TestBase, + with_rw_directory, + with_packs_rw, + fixture_path + ) from gitdb.stream import DeltaApplyReader from gitdb.pack import ( - PackEntity, - PackIndexFile, - PackFile - ) + PackEntity, + PackIndexFile, + PackFile + ) from gitdb.base import ( - OInfo, - OStream, - ) + OInfo, + OStream, + ) from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation @@ -35,213 +35,213 @@ #{ Utilities def bin_sha_from_filename(filename): - return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) #} END utilities class TestPack(TestBase): - - packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) - packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) - packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) - packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) - packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) - packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - - - def _assert_index_file(self, index, version, size): - assert index.packfile_checksum() != index.indexfile_checksum() - assert len(index.packfile_checksum()) == 20 - assert len(index.indexfile_checksum()) == 20 - assert index.version() == version - assert index.size() == size - assert len(index.offsets()) == size - - # get all data of all objects - for oidx in xrange(index.size()): - sha = index.sha(oidx) - assert oidx == index.sha_to_index(sha) - - entry = index.entry(oidx) - assert len(entry) == 3 - - assert entry[0] == index.offset(oidx) - assert entry[1] == sha - assert entry[2] == index.crc(oidx) - - # verify partial sha - for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l], l*2) == oidx - - # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - - - def _assert_pack_file(self, pack, version, size): - assert pack.version() == 2 - assert pack.size() == size - assert len(pack.checksum()) == 20 - - num_obj = 0 - for obj in pack.stream_iter(): - num_obj += 1 - info = pack.info(obj.pack_offset) - stream = pack.stream(obj.pack_offset) - - assert info.pack_offset == stream.pack_offset - assert info.type_id == stream.type_id - assert hasattr(stream, 'read') - - # it should be possible to read from both streams - assert obj.read() == stream.read() - - streams = pack.collect_streams(obj.pack_offset) - assert streams - - # read the stream - try: - dstream = DeltaApplyReader.new(streams) - except ValueError: - # ignore these, old git versions use only ref deltas, - # which we havent resolved ( as we are without an index ) - # Also ignore non-delta streams - continue - # END get deltastream - - # read all - data = dstream.read() - assert len(data) == dstream.size - - # test seek - dstream.seek(0) - assert dstream.read() == data - - - # read chunks - # NOTE: the current implementation is safe, it basically transfers - # all calls to the underlying memory map - - # END for each object - assert num_obj == size - - - def test_pack_index(self): - # check version 1 and 2 - for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): - index = PackIndexFile(indexfile) - self._assert_index_file(index, version, size) - # END run tests - - def test_pack(self): - # there is this special version 3, but apparently its like 2 ... - for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): - pack = PackFile(packfile) - self._assert_pack_file(pack, version, size) - # END for each pack to test - - @with_rw_directory - def test_pack_entity(self, rw_dir): - pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2), - (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): - packfile, version, size = packinfo - indexfile, version, size = indexinfo - entity = PackEntity(packfile) - assert entity.pack().path() == packfile - assert entity.index().path() == indexfile - pack_objs.extend(entity.stream_iter()) - - count = 0 - for info, stream in izip(entity.info_iter(), entity.stream_iter()): - count += 1 - assert info.binsha == stream.binsha - assert len(info.binsha) == 20 - assert info.type_id == stream.type_id - assert info.size == stream.size - - # we return fully resolved items, which is implied by the sha centric access - assert not info.type_id in delta_types - - # try all calls - assert len(entity.collect_streams(info.binsha)) - oinfo = entity.info(info.binsha) - assert isinstance(oinfo, OInfo) - assert oinfo.binsha is not None - ostream = entity.stream(info.binsha) - assert isinstance(ostream, OStream) - assert ostream.binsha is not None - - # verify the stream - try: - assert entity.is_valid_stream(info.binsha, use_crc=True) - except UnsupportedOperation: - pass - # END ignore version issues - assert entity.is_valid_stream(info.binsha, use_crc=False) - # END for each info, stream tuple - assert count == size - - # END for each entity - - # pack writing - write all packs into one - # index path can be None - pack_path = tempfile.mktemp('', "pack", rw_dir) - index_path = tempfile.mktemp('', 'index', rw_dir) - iteration = 0 - def rewind_streams(): - for obj in pack_objs: - obj.stream.seek(0) - #END utility - for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): - pfile = open(ppath, 'wb') - iwrite = None - if ipath: - ifile = open(ipath, 'wb') - iwrite = ifile.write - #END handle ip - - # make sure we rewind the streams ... we work on the same objects over and over again - if iteration > 0: - rewind_streams() - #END rewind streams - iteration += 1 - - pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) - pfile.close() - assert os.path.getsize(ppath) > 100 - - # verify pack - pf = PackFile(ppath) - assert pf.size() == len(pack_objs) - assert pf.version() == PackFile.pack_version_default - assert pf.checksum() == pack_sha - - # verify index - if ipath is not None: - ifile.close() - assert os.path.getsize(ipath) > 100 - idx = PackIndexFile(ipath) - assert idx.version() == PackIndexFile.index_version_default - assert idx.packfile_checksum() == pack_sha - assert idx.indexfile_checksum() == index_sha - assert idx.size() == len(pack_objs) - #END verify files exist - #END for each packpath, indexpath pair - - # verify the packs throughly - rewind_streams() - entity = PackEntity.create(pack_objs, rw_dir) - count = 0 - for info in entity.info_iter(): - count += 1 - for use_crc in range(2): - assert entity.is_valid_stream(info.binsha, use_crc) - # END for each crc mode - #END for each info - assert count == len(pack_objs) - - - def test_pack_64(self): - # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets - # of course without really needing such a huge pack - raise SkipTest() + + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) + + + def _assert_index_file(self, index, version, size): + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size + assert len(index.offsets()) == size + + # get all data of all objects + for oidx in xrange(index.size()): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4,8,11,17,20): + assert index.partial_sha_to_index(sha[:l], l*2) == oidx + + # END for each object index in indexfile + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) + + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = DeltaApplyReader.new(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we havent resolved ( as we are without an index ) + # Also ignore non-delta streams + continue + # END get deltastream + + # read all + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + + + # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map + + # END for each object + assert num_obj == size + + + def test_pack_index(self): + # check version 1 and 2 + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_objs = list() + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + pack_objs.extend(entity.stream_iter()) + + count = 0 + for info, stream in izip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # try all calls + assert len(entity.collect_streams(info.binsha)) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None + + # verify the stream + try: + assert entity.is_valid_stream(info.binsha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.binsha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path = tempfile.mktemp('', "pack", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + iteration = 0 + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + #END utility + for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + pfile = open(ppath, 'wb') + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write + #END handle ip + + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + rewind_streams() + #END rewind streams + iteration += 1 + + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pfile.close() + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == pack_sha + + # verify index + if ipath is not None: + ifile.close() + assert os.path.getsize(ipath) > 100 + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) + #END verify files exist + #END for each packpath, indexpath pair + + # verify the packs throughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in range(2): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + #END for each info + assert count == len(pack_objs) + + + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + raise SkipTest() diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 523f77056..6dc27463c 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -4,24 +4,24 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( - TestBase, - DummyStream, - Sha1Writer, - make_bytes, - make_object, - fixture_path - ) + TestBase, + DummyStream, + Sha1Writer, + make_bytes, + make_object, + fixture_path + ) from gitdb import * from gitdb.util import ( - NULL_HEX_SHA, - hex_to_bin - ) + NULL_HEX_SHA, + hex_to_bin + ) from gitdb.util import zlib from gitdb.typ import ( - str_blob_type - ) + str_blob_type + ) import time import tempfile @@ -31,124 +31,124 @@ class TestStream(TestBase): - """Test stream classes""" - - data_sizes = (15, 10000, 1000*1024+512) - - def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): - """Make stream tests - the orig_stream is seekable, allowing it to be - rewound and reused - :param cdata: the data we expect to read from stream, the contents - :param rewind_stream: function called to rewind the stream to make it ready - for reuse""" - ns = 10 - assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) - - # read in small steps - ss = len(cdata) / ns - for i in range(ns): - data = stream.read(ss) - chunk = cdata[i*ss:(i+1)*ss] - assert data == chunk - # END for each step - rest = stream.read() - if rest: - assert rest == cdata[-len(rest):] - # END handle rest - - if isinstance(stream, DecompressMemMapReader): - assert len(stream.data()) == stream.compressed_bytes_read() - # END handle special type - - rewind_stream(stream) - - # read everything - rdata = stream.read() - assert rdata == cdata - - if isinstance(stream, DecompressMemMapReader): - assert len(stream.data()) == stream.compressed_bytes_read() - # END handle special type - - def test_decompress_reader(self): - for close_on_deletion in range(2): - for with_size in range(2): - for ds in self.data_sizes: - cdata = make_bytes(ds, randomize=False) - - # zdata = zipped actual data - # cdata = original content data - - # create reader - if with_size: - # need object data - zdata = zlib.compress(make_object(str_blob_type, cdata)) - type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) - assert size == len(cdata) - assert type == str_blob_type - - # even if we don't set the size, it will be set automatically on first read - test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) - assert test_reader._s == len(cdata) - else: - # here we need content data - zdata = zlib.compress(cdata) - reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) - assert reader._s == len(cdata) - # END get reader - - self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) - - # put in a dummy stream for closing - dummy = DummyStream() - reader._m = dummy - - assert not dummy.closed - del(reader) - assert dummy.closed == close_on_deletion - # END for each datasize - # END whether size should be used - # END whether stream should be closed when deleted - - def test_sha_writer(self): - writer = Sha1Writer() - assert 2 == writer.write("hi") - assert len(writer.sha(as_hex=1)) == 40 - assert len(writer.sha(as_hex=0)) == 20 - - # make sure it does something ;) - prev_sha = writer.sha() - writer.write("hi again") - assert writer.sha() != prev_sha - - def test_compressed_writer(self): - for ds in self.data_sizes: - fd, path = tempfile.mkstemp() - ostream = FDCompressedSha1Writer(fd) - data = make_bytes(ds, randomize=False) - - # for now, just a single write, code doesn't care about chunking - assert len(data) == ostream.write(data) - ostream.close() - - # its closed already - self.failUnlessRaises(OSError, os.close, fd) - - # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) - written_data = os.read(fd, os.path.getsize(path)) - assert len(written_data) == os.path.getsize(path) - os.close(fd) - assert written_data == zlib.compress(data, 1) # best speed - - os.remove(path) - # END for each os - - def test_decompress_reader_special_case(self): - odb = LooseObjectDB(fixture_path('objects')) - ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) - - # if there is a bug, we will be missing one byte exactly ! - data = ostream.read() - assert len(data) == ostream.size - + """Test stream classes""" + + data_sizes = (15, 10000, 1000*1024+512) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) / ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i*ss:(i+1)*ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(str_blob_type, cdata)) + type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert type == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write("hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write("hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + + # its closed already + self.failUnlessRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 90f4156b9..35f9f44a7 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -8,98 +8,98 @@ from lib import TestBase from gitdb.util import ( - to_hex_sha, - to_bin_sha, - NULL_HEX_SHA, - LockedFD - ) + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, + LockedFD + ) - + class TestUtils(TestBase): - def test_basics(self): - assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA - assert len(to_bin_sha(NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA - - def _cmp_contents(self, file_path, data): - # raise if data from file at file_path - # does not match data string - fp = open(file_path, "rb") - try: - assert fp.read() == data - finally: - fp.close() - - def test_lockedfd(self): - my_file = tempfile.mktemp() - orig_data = "hello" - new_data = "world" - my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data) - my_file_fp.close() - - try: - lfd = LockedFD(my_file) - lockfilepath = lfd._lockfilepath() - - # cannot end before it was started - self.failUnlessRaises(AssertionError, lfd.rollback) - self.failUnlessRaises(AssertionError, lfd.commit) - - # open for writing - assert not os.path.isfile(lockfilepath) - wfd = lfd.open(write=True) - assert lfd._fd is wfd - assert os.path.isfile(lockfilepath) - - # write data and fail - os.write(wfd, new_data) - lfd.rollback() - assert lfd._fd is None - self._cmp_contents(my_file, orig_data) - assert not os.path.isfile(lockfilepath) - - # additional call doesnt fail - lfd.commit() - lfd.rollback() - - # test reading - lfd = LockedFD(my_file) - rfd = lfd.open(write=False) - assert os.read(rfd, len(orig_data)) == orig_data - - assert os.path.isfile(lockfilepath) - # deletion rolls back - del(lfd) - assert not os.path.isfile(lockfilepath) - - - # write data - concurrently - lfd = LockedFD(my_file) - olfd = LockedFD(my_file) - assert not os.path.isfile(lockfilepath) - wfdstream = lfd.open(write=True, stream=True) # this time as stream - assert os.path.isfile(lockfilepath) - # another one fails - self.failUnlessRaises(IOError, olfd.open) - - wfdstream.write(new_data) - lfd.commit() - assert not os.path.isfile(lockfilepath) - self._cmp_contents(my_file, new_data) - - # could test automatic _end_writing on destruction - finally: - os.remove(my_file) - # END final cleanup - - # try non-existing file for reading - lfd = LockedFD(tempfile.mktemp()) - try: - lfd.open(write=False) - except OSError: - assert not os.path.exists(lfd._lockfilepath()) - else: - self.fail("expected OSError") - # END handle exceptions + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + fp = open(file_path, "rb") + try: + assert fp.read() == data + finally: + fp.close() + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + my_file_fp = open(my_file, "wb") + my_file_fp.write(orig_data) + my_file_fp.close() + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.failUnlessRaises(AssertionError, lfd.rollback) + self.failUnlessRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesnt fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.failUnlessRaises(IOError, olfd.open) + + wfdstream.write(new_data) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/gitdb/util.py b/gitdb/util.py index 013f5fc78..1662b662d 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -13,28 +13,28 @@ # in py 2.4, StringIO is only StringI, without write support. # Hence we must use the python implementation for this if sys.version_info[1] < 5: - from StringIO import StringIO + from StringIO import StringIO # END handle python 2.4 try: - import async.mod.zlib as zlib + import async.mod.zlib as zlib except ImportError: - import zlib + import zlib # END try async zlib from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer + ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. if sys.version_info[1] < 6: - mman = StaticWindowMapManager() + mman = StaticWindowMapManager() else: - mman = SlidingWindowMapManager() + mman = SlidingWindowMapManager() #END handle mman try: @@ -43,19 +43,19 @@ import sha try: - from struct import unpack_from + from struct import unpack_from except ImportError: - from struct import unpack, calcsize - __calcsize_cache = dict() - def unpack_from(fmt, data, offset=0): - try: - size = __calcsize_cache[fmt] - except KeyError: - size = calcsize(fmt) - __calcsize_cache[fmt] = size - # END exception handling - return unpack(fmt, data[offset : offset + size]) - # END own unpack_from implementation + from struct import unpack, calcsize + __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset : offset + size]) + # END own unpack_from implementation #{ Globals @@ -100,25 +100,25 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... class _RandomAccessStringIO(object): - """Wrapper to provide required functionality in case memory maps cannot or may - not be used. This is only really required in python 2.4""" - __slots__ = '_sio' - - def __init__(self, buf=''): - self._sio = StringIO(buf) - - def __getattr__(self, attr): - return getattr(self._sio, attr) - - def __len__(self): - return len(self.getvalue()) - - def __getitem__(self, i): - return self.getvalue()[i] - - def __getslice__(self, start, end): - return self.getvalue()[start:end] - + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = StringIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + #} END compatibility stuff ... #{ Routines @@ -134,85 +134,85 @@ def make_sha(source=''): return sha1 def allocate_memory(size): - """:return: a file-protocol accessible memory block of the given size""" - if size == 0: - return _RandomAccessStringIO('') - # END handle empty chunks gracefully - - try: - return mmap.mmap(-1, size) # read-write by default - except EnvironmentError: - # setup real memory instead - # this of course may fail if the amount of memory is not available in - # one chunk - would only be the case in python 2.4, being more likely on - # 32 bit systems. - return _RandomAccessStringIO("\0"*size) - # END handle memory allocation - + """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessStringIO('') + # END handle empty chunks gracefully + + try: + return mmap.mmap(-1, size) # read-write by default + except EnvironmentError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return _RandomAccessStringIO("\0"*size) + # END handle memory allocation + def file_contents_ro(fd, stream=False, allow_mmap=True): - """:return: read-only contents of the file represented by the file descriptor fd - - :param fd: file descriptor opened for reading - :param stream: if False, random access is provided, otherwise the stream interface - is provided. - :param allow_mmap: if True, its allowed to map the contents into memory, which - allows large files to be handled and accessed efficiently. The file-descriptor - will change its position if this is False""" - try: - if allow_mmap: - # supports stream and random access - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - except EnvironmentError: - # python 2.4 issue, 0 wants to be the actual size - return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) - # END handle python 2.4 - except OSError: - pass - # END exception handling - - # read manully - contents = os.read(fd, os.fstat(fd).st_size) - if stream: - return _RandomAccessStringIO(contents) - return contents - + """:return: read-only contents of the file represented by the file descriptor fd + + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except EnvironmentError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 + except OSError: + pass + # END exception handling + + # read manully + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return _RandomAccessStringIO(contents) + return contents + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): - """Get the file contents at filepath as fast as possible - - :return: random access compatible memory of the given filepath - :param stream: see ``file_contents_ro`` - :param allow_mmap: see ``file_contents_ro`` - :param flags: additional flags to pass to os.open - :raise OSError: If the file could not be opened - - **Note** for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object - databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) - try: - return file_contents_ro(fd, stream, allow_mmap) - finally: - close(fd) - # END assure file is closed - + """Get the file contents at filepath as fast as possible + + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + def sliding_ro_buffer(filepath, flags=0): - """ - :return: a buffer compatible object which uses our mapped memory manager internally - ready to read the whole given filepath""" - return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) - + """ + :return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): - """:return: hexified version of sha""" - if len(sha) == 40: - return sha - return bin_to_hex(sha) - + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + def to_bin_sha(sha): - if len(sha) == 20: - return sha - return hex_to_bin(sha) + if len(sha) == 20: + return sha + return hex_to_bin(sha) #} END routines @@ -221,162 +221,162 @@ def to_bin_sha(sha): #{ Utilities class LazyMixin(object): - """ - Base class providing an interface to lazily retrieve attribute values upon - first access. If slots are used, memory will only be reserved once the attribute - is actually accessed and retrieved the first time. All future accesses will - return the cached value as stored in the Instance's dict or slot. - """ - - __slots__ = tuple() - - def __getattr__(self, attr): - """ - Whenever an attribute is requested that we do not know, we allow it - to be created and set. Next time the same attribute is reqeusted, it is simply - returned from our dict/slots. """ - self._set_cache_(attr) - # will raise in case the cache was not created - return object.__getattribute__(self, attr) - - def _set_cache_(self, attr): - """ - This method should be overridden in the derived class. - It should check whether the attribute named by attr can be created - and cached. Do nothing if you do not know the attribute or call your subclass - - The derived class may create as many additional attributes as it deems - necessary in case a git command returns more information than represented - in the single attribute.""" - pass - - + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is reqeusted, it is simply + returned from our dict/slots. """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ + This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + + class LockedFD(object): - """ - This class facilitates a safe read and write operation to a file on disk. - If we write to 'file', we obtain a lock file at 'file.lock' and write to - that instead. If we succeed, the lock file will be renamed to overwrite - the original file. - - When reading, we obtain a lock file, but to prevent other writers from - succeeding while we are reading the file. - - This type handles error correctly in that it will assure a consistent state - on destruction. - - **note** with this setup, parallel reading is not possible""" - __slots__ = ("_filepath", '_fd', '_write') - - def __init__(self, filepath): - """Initialize an instance with the givne filepath""" - self._filepath = filepath - self._fd = None - self._write = None # if True, we write a file - - def __del__(self): - # will do nothing if the file descriptor is already closed - if self._fd is not None: - self.rollback() - - def _lockfilepath(self): - return "%s.lock" % self._filepath - - def open(self, write=False, stream=False): - """ - Open the file descriptor for reading or writing, both in binary mode. - - :param write: if True, the file descriptor will be opened for writing. Other - wise it will be opened read-only. - :param stream: if True, the file descriptor will be wrapped into a simple stream - object which supports only reading or writing - :return: fd to read from or write to. It is still maintained by this instance - and must not be closed directly - :raise IOError: if the lock could not be retrieved - :raise OSError: If the actual file could not be opened for reading - - **note** must only be called once""" - if self._write is not None: - raise AssertionError("Called %s multiple times" % self.open) - - self._write = write - - # try to open the lock file - binary = getattr(os, 'O_BINARY', 0) - lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary - try: - fd = os.open(self._lockfilepath(), lockmode, 0600) - if not write: - os.close(fd) - else: - self._fd = fd - # END handle file descriptor - except OSError: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) - # END handle lock retrieval - - # open actual file if required - if self._fd is None: - # we could specify exlusive here, as we obtained the lock anyway - try: - self._fd = os.open(self._filepath, os.O_RDONLY | binary) - except: - # assure we release our lockfile - os.remove(self._lockfilepath()) - raise - # END handle lockfile - # END open descriptor for reading - - if stream: - # need delayed import - from stream import FDStream - return FDStream(self._fd) - else: - return self._fd - # END handle stream - - def commit(self): - """When done writing, call this function to commit your changes into the - actual file. - The file descriptor will be closed, and the lockfile handled. - - **Note** can be called multiple times""" - self._end_writing(successful=True) - - def rollback(self): - """Abort your operation without any changes. The file descriptor will be - closed, and the lock released. - - **Note** can be called multiple times""" - self._end_writing(successful=False) - - def _end_writing(self, successful=True): - """Handle the lock according to the write mode """ - if self._write is None: - raise AssertionError("Cannot end operation if it wasn't started yet") - - if self._fd is None: - return - - os.close(self._fd) - self._fd = None - - lockfile = self._lockfilepath() - if self._write and successful: - # on windows, rename does not silently overwrite the existing one - if sys.platform == "win32": - if isfile(self._filepath): - os.remove(self._filepath) - # END remove if exists - # END win32 special handling - os.rename(lockfile, self._filepath) - - # assure others can at least read the file - the tmpfile left it at rw-- - # We may also write that file, on windows that boils down to a remove- - # protection as well - chmod(self._filepath, 0644) - else: - # just delete the file so far, we failed - os.remove(lockfile) - # END successful handling + """ + This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + **note** with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """ + Open the file descriptor for reading or writing, both in binary mode. + + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + + **note** must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode, 0600) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exlusive here, as we obtained the lock anyway + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + os.remove(self._lockfilepath()) + raise + # END handle lockfile + # END open descriptor for reading + + if stream: + # need delayed import + from stream import FDStream + return FDStream(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + + **Note** can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + + **Note** can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if isfile(self._filepath): + os.remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, 0644) + else: + # just delete the file so far, we failed + os.remove(lockfile) + # END successful handling #} END utilities From 6099ac8e04a8f734555efe4e78ad766d9a6fb70a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:23:09 +0200 Subject: [PATCH 0205/1790] Added travis CI support --- .travis.yml | 8 ++++++++ README.rst | 8 +++++--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..e7d5214c1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.5" + - "2.6" + - "2.7" + - "pypy" + +script: nosetests diff --git a/README.rst b/README.rst index 30bff0ded..84feb373c 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,8 @@ Although memory maps have many advantages, they represent a very limited system Overview ######## +.. image:: https://travis-ci.org/Byron/smmap.svg?branch=master :target: https://travis-ci.org/Byron/smmap + Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. @@ -58,17 +60,17 @@ The project is home on github at `https://github.com/Byron/smmap Date: Sun, 4 May 2014 16:29:43 +0200 Subject: [PATCH 0206/1790] pypy deactivated, it doesn't yet work --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e7d5214c1..563797656 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: python python: + - "2.4" - "2.5" - "2.6" - "2.7" - - "pypy" + # - "pypy" - no getrefcount script: nosetests From 616e9ceaf917e4d8f3cf2c145401b8069ce307dd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:33:55 +0200 Subject: [PATCH 0207/1790] Oh, travis doesn't support older python versions, fair enough --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 563797656..fdba549d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python python: - - "2.4" - - "2.5" - "2.6" - "2.7" # - "pypy" - no getrefcount From a4eddfe72b442666ba3acdf89929e9b65de45f45 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:38:55 +0200 Subject: [PATCH 0208/1790] Added initial configuration of gitdb --- README.rst | 3 +++ gitdb/.travis.yml | 7 +++++++ gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 gitdb/.travis.yml diff --git a/README.rst b/README.rst index 753eb701c..68f71ff61 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,9 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= + +.. image:: https://travis-ci.org/Byron/gitdb.svg?branch=master :target: https://travis-ci.org/Byron/gitdb + https://github.com/gitpython-developers/gitdb/issues LICENSE diff --git a/gitdb/.travis.yml b/gitdb/.travis.yml new file mode 100644 index 000000000..5c8791d90 --- /dev/null +++ b/gitdb/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.6" + - "2.7" + # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) + +script: nosetests diff --git a/gitdb/ext/async b/gitdb/ext/async index 571412931..90326fb86 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 571412931829200aff06a44b9c5524e122e524e9 +Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 1b3ab5598..616e9ceaf 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 1b3ab5598e93369282502d049d64cb2ca12839cb +Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd From e2c94bf6983b378f99db175cd7810986cf338c32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:43:26 +0200 Subject: [PATCH 0209/1790] argh, one level too low, didn't see it in sublime --- gitdb/.travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gitdb/.travis.yml => .travis.yml (100%) diff --git a/gitdb/.travis.yml b/.travis.yml similarity index 100% rename from gitdb/.travis.yml rename to .travis.yml From 39de1127459b73b862f2b779bb4565ad6b4bd625 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:44:48 +0200 Subject: [PATCH 0210/1790] Fixed travis build status url --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 68f71ff61..0fc0534e6 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -.. image:: https://travis-ci.org/Byron/gitdb.svg?branch=master :target: https://travis-ci.org/Byron/gitdb +.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb https://github.com/gitpython-developers/gitdb/issues From cf1cf469f315df00ce7b9d47693008cd2fa1b56a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 May 2014 09:31:15 +0200 Subject: [PATCH 0211/1790] Get starten on py3.3 compatability This is likely to fail, but lets see. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fdba549d4..d8640a08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.3" # - "pypy" - no getrefcount script: nosetests From 32d34eba0412770c382a1428a0ebb221dbba4187 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 May 2014 09:55:32 +0200 Subject: [PATCH 0212/1790] Test with python 3.3 as well Have to start making them compatible at some point. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5c8791d90..ff263f20d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.3" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) script: nosetests From 2d3b5a303a65d6f80b84e63a1b3cf4b670c81f9a Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 15:38:03 +1000 Subject: [PATCH 0213/1790] Initial work for supporting python 3 (>= 3.3). Signed-off-by: David Black --- smmap/__init__.py | 4 ++-- smmap/buf.py | 8 ++++---- smmap/mman.py | 20 +++++++++++--------- smmap/test/test_buf.py | 6 +++--- smmap/test/test_mman.py | 8 ++++---- smmap/test/test_tutorial.py | 2 +- smmap/test/test_util.py | 8 ++++---- smmap/util.py | 13 ++++++++++--- 8 files changed, 39 insertions(+), 30 deletions(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index a10cd5c99..879ebea24 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -7,5 +7,5 @@ __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience -from mman import * -from buf import * +from .mman import * +from .buf import * diff --git a/smmap/buf.py b/smmap/buf.py index 255c6b54d..3917ee8be 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import WindowCursor +from .mman import WindowCursor import sys @@ -21,7 +21,7 @@ class SlidingWindowMapBuffer(object): ) - def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): """Initalize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access If None, you have call begin_access before using the buffer and provide a cursor @@ -61,7 +61,7 @@ def __getslice__(self, i, j): assert c.is_valid() if i < 0: i = self._size + i - if j == sys.maxint: + if j == sys.maxsize: j = self._size if j < 0: j = self._size + j @@ -86,7 +86,7 @@ def __getslice__(self, i, j): # END fast or slow path #{ Interface - def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. diff --git a/smmap/mman.py b/smmap/mman.py index 97c42c5bb..9cc251f0f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,15 +1,17 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -from util import ( +from .util import ( MapWindow, MapRegion, MapRegionList, is_64_bit, - align_to_mmap + align_to_mmap, + string_types, ) from weakref import ref import sys from sys import getrefcount +from functools import reduce __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] #{ Utilities @@ -218,7 +220,7 @@ def fd(self): **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), basestring): + if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() @@ -256,7 +258,7 @@ class StaticWindowMapManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): """initialize the manager with the given parameters. :param window_size: if -1, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size @@ -306,7 +308,7 @@ def _collect_lru_region(self, size): while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None lru_list = None - for regions in self._fdict.itervalues(): + for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and @@ -343,7 +345,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): r = a[0] else: try: - r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxsize, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -405,7 +407,7 @@ def num_file_handles(self): def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) def window_size(self): """:return: size of each window when allocating new regions""" @@ -445,7 +447,7 @@ def force_map_handle_removal_win(self, base_path): #END early bailout num_closed = 0 - for path, rlist in self._fdict.iteritems(): + for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: region._mf.close() @@ -473,7 +475,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): __slots__ = tuple() - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxsize): """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 4bdcb76f5..d40da1479 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager from smmap.buf import * @@ -22,8 +22,8 @@ def test_basics(self): # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 46429a419..0929583c9 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.mman import * from smmap.mman import WindowCursor @@ -66,7 +66,7 @@ def test_memory_manager(self): man._collect_lru_region(10) # doesn't fail if we overallocate - assert man._collect_lru_region(sys.maxint) == 0 + assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") @@ -80,9 +80,9 @@ def test_memory_manager(self): assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) + self.assertRaises(ValueError, c.path) else: - self.failUnlessRaises(ValueError, c.fd) + self.assertRaises(ValueError, c.fd) #END handle value error #END for each input os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index 4e1a5764b..ad1a9c0b5 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,4 +1,4 @@ -from lib import TestBase +from .lib import TestBase class TestTutorial(TestBase): diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 2df0660be..a009bd9d2 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.util import * @@ -38,7 +38,7 @@ def test_window(self): assert wc.ofs == 1 and wc.size == maxsize # without maxsize - wc.extend_right_to(wr, sys.maxint) + wc.extend_right_to(wr, sys.maxsize) assert wc.ofs_end() == wr.ofs and wc.ofs == 1 # extend left @@ -46,7 +46,7 @@ def test_window(self): wr.extend_left_to(wc2, maxsize) assert wr.size == maxsize - wr.extend_left_to(wc2, sys.maxint) + wr.extend_left_to(wc2, sys.maxsize) assert wr.ofs == wc2.ofs_end() wc.align() @@ -68,7 +68,7 @@ def test_region(self): assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. # We only test on linux as it is inconsitent between the python versions diff --git a/smmap/util.py b/smmap/util.py index c6710b3fe..fee9ad591 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -19,6 +19,13 @@ #{ Utilities +def string_types(): + if sys.version_info[0] >= 3: + return str + else: + return basestring + + def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. @@ -34,7 +41,7 @@ def align_to_mmap(num, round_up): def is_64_bit(): """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxint > (1<<32) - 1 + return sys.maxsize > (1<<32) - 1 #}END utilities @@ -154,7 +161,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping finally: - if isinstance(path_or_fd, basestring): + if isinstance(path_or_fd, string_types()): os.close(fd) #END only close it if we opened it #END close file handle @@ -258,7 +265,7 @@ def path_or_fd(self): def file_size(self): """:return: size of file we manager""" if self._file_size is None: - if isinstance(self._path_or_fd, basestring): + if isinstance(self._path_or_fd, string_types()): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size From 575f4a895ea525ecfedbfc1dc4c6f032ad92ccdc Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 16:35:20 +1000 Subject: [PATCH 0214/1790] instead of writing a string to a buffer api (in python 3) write a buffer. Signed-off-by: David Black --- smmap/test/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 21e6c5a09..01f6cc918 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -22,7 +22,7 @@ def __init__(self, size, prefix=''): fp = open(self._path, "wb") fp.seek(size-1) - fp.write('1') + fp.write(b'1') fp.close() assert os.path.getsize(self.path) == size From ef3b36e6ae20e7bd6c31c71623a5e0019ba9eec9 Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 16:36:47 +1000 Subject: [PATCH 0215/1790] _need_compat_layer is not required in python 3. Signed-off-by: David Black --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index fee9ad591..f4ecd228c 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -104,7 +104,7 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[1] < 6 + _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset From afcf97e6cd6584c97ccfe3de57bdc58df0ba9f8a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 16 May 2014 08:31:09 -0700 Subject: [PATCH 0216/1790] Add tox.ini; make .travis.yml use it --- .gitignore | 1 + .travis.yml | 16 +++++++++------- tox.ini | 11 +++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 6cfb58df1..11852be02 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build/ coverage dist/ MANIFEST +.tox diff --git a/.travis.yml b/.travis.yml index d8640a08e..91f43283b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ language: python -python: - - "2.6" - - "2.7" - - "3.3" - # - "pypy" - no getrefcount - -script: nosetests +env: + - TOXENV=py26 + - TOXENV=py27 + - TOXENV=py33 + - TOXENV=py34 +install: + - pip install tox +script: + - tox diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..80190d0a3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py33, py34 + +[testenv] +commands = nosetests +deps = nose From f0680101739da3435bbae0139765bb5ce65bcb92 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 19 May 2014 23:51:50 +0200 Subject: [PATCH 0217/1790] Added coverage reporting. In the process, I removed tox as it made things so much more complex for me. --- .coveragerc | 10 ++++++ .gitignore | 1 + .travis.yml | 15 ++++----- README.rst => README.md | 70 ++++++++++++++++++++++------------------- tox.ini | 11 ------- 5 files changed, 57 insertions(+), 50 deletions(-) create mode 100644 .coveragerc rename README.rst => README.md (71%) delete mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..e61d27ba9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = smmap + +; to make nosetests happy +[report] +omit = + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 11852be02..2081aafd2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ .coverage coverage +cover/ dist/ MANIFEST .tox diff --git a/.travis.yml b/.travis.yml index 91f43283b..c63e5e325 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ language: python -env: - - TOXENV=py26 - - TOXENV=py27 - - TOXENV=py33 - - TOXENV=py34 +python: + - "2.6" + - "2.7" + - "3.3" install: - - pip install tox + - pip install coveralls script: - - tox + - nosetests --with-coverage +after_success: + - coveralls diff --git a/README.rst b/README.md similarity index 71% rename from README.rst rename to README.md index 84feb373c..c056ed1c1 100644 --- a/README.rst +++ b/README.md @@ -1,15 +1,15 @@ -########### -Motivation -########### +## Motivation + When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. -######## -Overview -######## -.. image:: https://travis-ci.org/Byron/smmap.svg?branch=master :target: https://travis-ci.org/Byron/smmap + +## Overview + +[![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) +[![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. @@ -21,42 +21,49 @@ Although the library can be used most efficiently with its native interface, a B For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. -############# -Prerequisites -############# + + +## Prerequisites + * Python 2.4, 2.5 or 2.6 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. -########### -Limitations -########### + + +## Limitations + * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. * It wasn't tested on python 2.7 and 3.x. -################ -Installing smmap -################ -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + +## Installing smmap + +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively: - $ easy_install smmap - # or - $ pip install smmap +```bash +$ easy_install smmap +# or +$ pip install smmap +``` As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. -If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: +If you have downloaded the source archive, the package can be installed by running the `setup.py` script: - $ python setup.py install +```bash +$ python setup.py install +``` + +It is advised to have a look at the **Usage Guide** for a brief introduction on the different database implementations. + -It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. -################## -Homepage and Links -################## -The project is home on github at `https://github.com/Byron/smmap `_. +## Homepage and Links + +The project is home on github at https://github.com/Byron/smmap . The latest source can be cloned from github as well: @@ -72,10 +79,9 @@ Issues can be filed on github: * https://github.com/Byron/smmap/issues -################### -License Information -################### + + +## License Information + *smmap* is licensed under the New BSD License. -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools -.. _pip: http://www.pip-installer.org/en/latest/ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 80190d0a3..000000000 --- a/tox.ini +++ /dev/null @@ -1,11 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py26, py27, py33, py34 - -[testenv] -commands = nosetests -deps = nose From 403ad7816e2c88be78606cfee2d9944f535e7ed7 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 13 Jun 2014 22:24:23 -0700 Subject: [PATCH 0218/1790] Delay importing sys.getrefcount until needed This makes it possibly to at least install on PyPy Addresses: GH-4 ("pypy compatibility") --- smmap/mman.py | 1 - smmap/util.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9cc251f0f..637a2844d 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -10,7 +10,6 @@ from weakref import ref import sys -from sys import getrefcount from functools import reduce __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] diff --git a/smmap/util.py b/smmap/util.py index f4ecd228c..ec86cbf16 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,8 +12,6 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -from sys import getrefcount - __all__ = [ "align_to_mmap", "is_64_bit", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] @@ -210,6 +208,7 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" + from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 @@ -256,6 +255,7 @@ def __init__(self, path_or_fd): def client_count(self): """:return: amount of clients which hold a reference to this instance""" + from sys import getrefcount return getrefcount(self)-3 def path_or_fd(self): From f3d3502bef512ed48581f69fee9b655a91e06db8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Sun, 15 Jun 2014 23:33:04 -0700 Subject: [PATCH 0219/1790] Change / to // (integer division) in several places This fixes a bunch of bugs and test failures in Python 3, which uses "true division" for / --- smmap/test/test_mman.py | 6 +++--- smmap/test/test_util.py | 2 +- smmap/util.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 0929583c9..e0516b21c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -95,8 +95,8 @@ def test_memman_operation(self): fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size @@ -110,7 +110,7 @@ def test_memman_operation(self): base_offset = 5000 # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() / 2 + size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() assert rr().client_count() == 2 # the manager and the cursor and us diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index a009bd9d2..8afba005e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -54,7 +54,7 @@ def test_window(self): def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size / 2 + half_size = fc.size // 2 rofs = align_to_mmap(4200, False) rfull = MapRegion(fc.path, 0, fc.size) rhalfofs = MapRegion(fc.path, rofs, fc.size) diff --git a/smmap/util.py b/smmap/util.py index ec86cbf16..0d8385db2 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -31,7 +31,7 @@ def align_to_mmap(num, round_up): :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; if round_up and (res != num): res += ALLOCATIONGRANULARITY #END handle size From 707885bebe6cefa45dae7dd13dadac5b2b98d16d Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:41:25 -0700 Subject: [PATCH 0220/1790] .gitignore: Add *.egg-info --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2081aafd2..73b0b6188 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ cover/ dist/ MANIFEST .tox +*.egg-info From 23b79ea2838510028de43a6edb1b09246dca9a46 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:47:31 -0700 Subject: [PATCH 0221/1790] Add back tox.ini for tox This time with support for measuring coverage. Ability to test quickly across multiple Python versions is crucial for working on Python 3 compatibility... --- tox.ini | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..e0e196418 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py33, py34 + +[testenv] +commands = nosetests {posargs:--with-coverage --cover-package=smmap} +deps = + nose + nosexcover From 52fba74d78d3fcce53fb89c7b2c20b24197296a8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:33:41 -0700 Subject: [PATCH 0222/1790] Deal with lack of `buffer` in py3 --- smmap/mman.py | 1 + smmap/test/test_tutorial.py | 1 + smmap/util.py | 11 ++++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/smmap/mman.py b/smmap/mman.py index 637a2844d..ff32bbb51 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -6,6 +6,7 @@ is_64_bit, align_to_mmap, string_types, + buffer, ) from weakref import ref diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index ad1a9c0b5..ccc113b4f 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -44,6 +44,7 @@ def test_example(self): # its recommended not to create big slices when feeding the buffer # into consumers (e.g. struct or zlib). # Instead, either give the buffer directly, or use pythons buffer command. + from smmap.util import buffer buffer(c.buffer(), 1, 9) # first 9 bytes without copying them # you can query absolute offsets, and check whether an offset is included diff --git a/smmap/util.py b/smmap/util.py index 0d8385db2..54c6c45c3 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,11 +12,20 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -__all__ = [ "align_to_mmap", "is_64_bit", +__all__ = [ "align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities +try: + # Python 2 + buffer = buffer +except NameError: + # Python 3 has no `buffer`; only `memoryview` + def buffer(obj, offset, size): + return memoryview(obj[offset:offset+size]) + + def string_types(): if sys.version_info[0] >= 3: return str From cf515928248b7c34cb73b6e20edbdadebd086f96 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 07:55:21 -0700 Subject: [PATCH 0223/1790] Fix 2 instances of "containnig" => "containing" --- smmap/mman.py | 2 +- smmap/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ff32bbb51..7cbb535bd 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,4 +1,4 @@ -"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( MapWindow, MapRegion, diff --git a/smmap/util.py b/smmap/util.py index 54c6c45c3..c37dfdd31 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -1,4 +1,4 @@ -"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" import os import sys import mmap From c0b94c67b86b4250364a951f4865d714952f792a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 08:54:30 -0700 Subject: [PATCH 0224/1790] .travis.yml: Allow py33 to fail, add py34, etc. --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c63e5e325..d02eb6f5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,17 @@ language: python python: - - "2.6" - - "2.7" - - "3.3" + - 2.6 + - 2.7 + - 3.3 + - 3.4 install: - pip install coveralls script: - nosetests --with-coverage after_success: - coveralls +matrix: + allow_failures: + - python: 3.3 + - python: 3.4 + fast_finish: true From 33e6314941572a93b8031b1feecded22ef1f5f3c Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 08:49:04 -0700 Subject: [PATCH 0225/1790] Use bytes() instead of str() bytes() is more accurate and is actually correct in Python 3, whereas str() is incorrect in Python 3, because it's a Unicode string. --- smmap/buf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/smmap/buf.py b/smmap/buf.py index 3917ee8be..ba6f8ede7 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -5,6 +5,12 @@ __all__ = ["SlidingWindowMapBuffer"] +try: + bytes +except NameError: + bytes = str + + class SlidingWindowMapBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. @@ -73,7 +79,7 @@ def __getslice__(self, i, j): ofs = i # Keeping tokens in a list could possible be faster, but the list # overhead outweighs the benefits (tested) ! - md = str() + md = bytes() while l: c.use_region(ofs, l) assert c.is_valid() From 19917bdc3d30d7d3e6dba082ba132a86d0190b09 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 09:05:33 -0700 Subject: [PATCH 0226/1790] Fix typo: "optimial" => "optimal" --- smmap/test/test_buf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d40da1479..23a4fbbcd 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -76,7 +76,7 @@ def test_basics(self): for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case'), - (static_man, 'static optimial')): + (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi From 8fde5f3fb8711f866612c657d20c9d3e9cf59f41 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 09:02:30 -0700 Subject: [PATCH 0227/1790] Make __getitem__ handle slice for Python 3 Python 3 doesn't have __getslice__ instead it uses __getitem__ with a slice object. --- smmap/buf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smmap/buf.py b/smmap/buf.py index ba6f8ede7..2f27d4d01 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -51,6 +51,8 @@ def __len__(self): return self._size def __getitem__(self, i): + if isinstance(i, slice): + return self.__getslice__(i.start or 0, i.stop or self._size) c = self._c assert c.is_valid() if i < 0: From 651dfa8a359200b66e686998af2640efb54f16ad Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 10:48:43 -0700 Subject: [PATCH 0228/1790] Change / to // (integer division) in test_buf.py This fixes (the last!) test failure in Python 3, which uses "true division" for / --- smmap/test/test_buf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 23a4fbbcd..15dfb8238 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -10,9 +10,10 @@ man_optimal = SlidingWindowMapManager() -man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, - max_memory_size=TestBase.k_window_test_size/3, - max_open_handles=15) +man_worst_case = SlidingWindowMapManager( + window_size=TestBase.k_window_test_size // 100, + max_memory_size=TestBase.k_window_test_size // 3, + max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): From f75dcbcf5d10a639c031dd706fa7fcfa1784eecb Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 11:09:54 -0700 Subject: [PATCH 0229/1790] .travis.yml: Stop allowing failures for py3{3,4} --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d02eb6f5e..47cb41170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,3 @@ script: - nosetests --with-coverage after_success: - coveralls -matrix: - allow_failures: - - python: 3.3 - - python: 3.4 - fast_finish: true From fbbb3090ea1ca4ac3042acea7633241b0388f0b3 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 11:31:48 -0700 Subject: [PATCH 0230/1790] setup.py: Add Python 3 (and Python 2) classifiers --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 2e97e59c6..13d2063ab 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,12 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", ], long_description=long_description, ) From 55267119140f3828a24b4986600ed21a1808d6cc Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 12:02:10 -0700 Subject: [PATCH 0231/1790] setup.cfg: Specify that wheel is universal --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From d79c655a7bf281a39662f4a4562d76312b69515a Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 14:21:09 -0400 Subject: [PATCH 0232/1790] Update async to the Python 2 / 3 compatible version --- gitdb/ext/async | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/async b/gitdb/ext/async index 90326fb86..339024bfb 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 +Subproject commit 339024bfb1d0a2b091e63d7a7ea23a1c63189f5c From 72167492334b756ddeaa606274e9348a70734cdb Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 14:27:22 -0400 Subject: [PATCH 0233/1790] Update smmap to a Python 3 compatible version --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 616e9ceaf..552671191 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd +Subproject commit 55267119140f3828a24b4986600ed21a1808d6cc From 1c6f4c19289732bd13507eba9e54c9d692957137 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:35:24 -0400 Subject: [PATCH 0234/1790] Automated PEP 8 fixes --- gitdb/__init__.py | 4 +- gitdb/base.py | 205 ++++++++++--------- gitdb/db/base.py | 162 ++++++++------- gitdb/db/git.py | 34 ++-- gitdb/db/loose.py | 103 +++++----- gitdb/db/mem.py | 56 +++-- gitdb/db/pack.py | 88 ++++---- gitdb/db/ref.py | 18 +- gitdb/exc.py | 10 +- gitdb/fun.py | 230 ++++++++++----------- gitdb/pack.py | 394 ++++++++++++++++++------------------ gitdb/stream.py | 318 +++++++++++++++-------------- gitdb/test/__init__.py | 2 +- gitdb/test/db/lib.py | 80 ++++---- gitdb/test/db/test_git.py | 22 +- gitdb/test/db/test_loose.py | 15 +- gitdb/test/db/test_mem.py | 18 +- gitdb/test/db/test_pack.py | 28 +-- gitdb/test/db/test_ref.py | 28 ++- gitdb/test/lib.py | 39 ++-- gitdb/test/test_base.py | 28 +-- gitdb/test/test_example.py | 26 +-- gitdb/test/test_pack.py | 102 +++++----- gitdb/test/test_stream.py | 51 +++-- gitdb/test/test_util.py | 43 ++-- gitdb/util.py | 131 ++++++------ 26 files changed, 1108 insertions(+), 1127 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ff750d14c..847269a33 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -12,14 +12,14 @@ def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('async', 'smmap'): sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - + try: __import__(module) except ImportError: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) #END verify import #END handel imports - + #} END initialization _init_externals() diff --git a/gitdb/base.py b/gitdb/base.py index bad5f7472..a673c2376 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -13,183 +13,183 @@ type_to_type_id_map ) -__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', 'IStream', 'InvalidOInfo', 'InvalidOStream' ) #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provding information + """Carries information about an object in an ODB, provding information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. - + It can be accessed using tuple notation and using attribute access notation:: - + assert dbi[0] == dbi.binsha assert dbi[1] == dbi.type assert dbi[2] == dbi.size - + The type is designed to be as lighteight as possible.""" __slots__ = tuple() - + def __new__(cls, sha, type, size): return tuple.__new__(cls, (sha, type, size)) - + def __init__(self, *args): tuple.__init__(self) - - #{ Interface + + #{ Interface @property def binsha(self): """:return: our sha as binary, 20 bytes""" return self[0] - + @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" return bin_to_hex(self[0]) - + @property def type(self): return self[1] - + @property def type_id(self): return type_to_type_id_map[self[1]] - + @property def size(self): return self[2] #} END interface - - + + class OPackInfo(tuple): - """As OInfo, but provides a type_id property to retrieve the numerical type id, and + """As OInfo, but provides a type_id property to retrieve the numerical type id, and does not include a sha. - - Additionally, the pack_offset is the absolute offset into the packfile at which + + Additionally, the pack_offset is the absolute offset into the packfile at which all object information is located. The data_offset property points to the abosolute location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size): return tuple.__new__(cls, (packoffset,type, size)) - + def __init__(self, *args): tuple.__init__(self) - - #{ Interface - + + #{ Interface + @property def pack_offset(self): return self[0] - + @property def type(self): return type_id_to_type_map[self[1]] - + @property def type_id(self): return self[1] - + @property def size(self): return self[2] - + #} END interface - - + + class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, or the negative offset from the pack_offset, so that pack_offset - delta_info yields the pack offset of the base object""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, delta_info): return tuple.__new__(cls, (packoffset, type, size, delta_info)) - - #{ Interface + + #{ Interface @property def delta_info(self): return self[3] - #} END interface - - + #} END interface + + class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional + """Base for object streams retrieved from the database, providing additional information about the stream. Generally, ODB streams are read-only as objects are immutable""" __slots__ = tuple() - + def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - - + + def __init__(self, *args, **kwargs): tuple.__init__(self) - - #{ Stream Reader Interface - + + #{ Stream Reader Interface + def read(self, size=-1): return self[3].read(size) - + @property def stream(self): return self[3] - + #} END stream reader interface - - + + class ODeltaStream(OStream): """Uses size info of its stream, delaying reads""" - + def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - + #{ Stream Reader Interface - + @property def size(self): return self[3].size - + #} END stream reader interface - - + + class OPackStream(OPackInfo): """Next to pack object information, a stream outputting an undeltified base object is provided""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (packoffset, type, size, stream)) - - #{ Stream Reader Interface + + #{ Stream Reader Interface def read(self, size=-1): return self[3].read(size) - + @property def stream(self): return self[3] #} END stream reader interface - + class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, delta_info, stream): return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface + #{ Stream Reader Interface def read(self, size=-1): return self[4].read(size) - + @property def stream(self): return self[4] @@ -197,106 +197,106 @@ def stream(self): class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow + """Represents an input content stream to be fed into the ODB. It is mutable to allow the ODB to record information about the operations outcome right in this instance. - + It provides interfaces for the OStream and a StreamReader to allow the instance to blend in without prior conversion. - + The only method your content stream must support is 'read'""" __slots__ = tuple() - + def __new__(cls, type, size, stream, sha=None): return list.__new__(cls, (sha, type, size, stream, None)) - + def __init__(self, type, size, stream, sha=None): list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface + + #{ Interface @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" return bin_to_hex(self[0]) - + def _error(self): """:return: the error that occurred when processing the stream, or None""" return self[4] - + def _set_error(self, exc): """Set this input stream to the given exc, may be None to reset the error""" self[4] = exc - + error = property(_error, _set_error) - + #} END interface - + #{ Stream Reader Interface - + def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on + """Implements a simple stream reader interface, passing the read call on to our internal stream""" return self[3].read(size) - - #} END stream reader interface - + + #} END stream reader interface + #{ interface - + def _set_binsha(self, binsha): self[0] = binsha - + def _binsha(self): return self[0] - + binsha = property(_binsha, _set_binsha) - - + + def _type(self): return self[1] - + def _set_type(self, type): self[1] = type - + type = property(_type, _set_type) - + def _size(self): return self[2] - + def _set_size(self, size): self[2] = size - + size = property(_size, _set_size) - + def _stream(self): return self[3] - + def _set_stream(self, stream): self[3] = stream - + stream = property(_stream, _set_stream) - - #} END odb info interface - + + #} END odb info interface + class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in + """Carries information about a sha identifying an object which is invalid in the queried database. The exception attribute provides more information about the cause of the issue""" __slots__ = tuple() - + def __new__(cls, sha, exc): return tuple.__new__(cls, (sha, exc)) - + def __init__(self, sha, exc): tuple.__init__(self, (sha, exc)) - + @property def binsha(self): return self[0] - + @property def hexsha(self): return bin_to_hex(self[0]) - + @property def error(self): """:return: exception instance explaining the failure""" @@ -306,6 +306,5 @@ def error(self): class InvalidOStream(InvalidOInfo): """Carries information about an invalid ODB stream""" __slots__ = tuple() - -#} END ODB Bases +#} END ODB Bases diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 867e93a81..0eef1e5d5 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,20 +4,20 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, - join, - LazyMixin, - hex_to_bin - ) + pool, + join, + LazyMixin, + hex_to_bin +) from gitdb.exc import ( - BadObject, - AmbiguousObjectName - ) + BadObject, + AmbiguousObjectName +) from async import ( - ChannelThreadTask - ) + ChannelThreadTask +) from itertools import chain @@ -28,17 +28,17 @@ class ObjectDBR(object): """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" - + def __contains__(self, sha): return self.has_obj - - #{ Query Interface + + #{ Query Interface def has_object(self, sha): """ :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") - + def has_object_async(self, reader): """Return a reader yielding information about the membership of objects as identified by shas @@ -46,62 +46,62 @@ def has_object_async(self, reader): :return: async.Reader yielding tuples of (sha, bool) pairs which indicate whether the given sha exists in the database or not""" task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - + return pool.add_task(task) + def info(self, sha): """ :return: OInfo instance :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - + def info_async(self, reader): """Retrieve information of a multitude of objects asynchronously :param reader: Channel yielding the sha's of the objects of interest :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" task = ChannelThreadTask(reader, str(self.info_async), self.info) return pool.add_task(task) - + def stream(self, sha): """:return: OStream instance :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - + def stream_async(self, reader): """Retrieve the OStream of multiple objects :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to + + **Note:** depending on the system configuration, it might not be possible to read all OStreams at once. Instead, read them individually using reader.read(x) where x is small enough.""" # base implementation just uses the stream method repeatedly task = ChannelThreadTask(reader, str(self.stream_async), self.stream) return pool.add_task(task) - + def size(self): """:return: amount of objects in this database""" raise NotImplementedError() - + def sha_iter(self): """Return iterator yielding 20 byte shas for all objects in this data base""" raise NotImplementedError() - + #} END query interface - - + + class ObjectDBW(object): """Defines an interface to create objects in the database""" - + def __init__(self, *args, **kwargs): self._ostream = None - + #{ Edit Interface def set_ostream(self, stream): """ Adjusts the stream to which all data should be sent when storing new objects - + :param stream: if not None, the stream to use, if None the default stream will be used. :return: previously installed stream, or None if there was no override @@ -109,96 +109,96 @@ def set_ostream(self, stream): cstream = self._ostream self._ostream = stream return cstream - + def ostream(self): """ :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream - + def store(self, istream): """ Create a new object in the database :return: the input istream object with its sha set to its corresponding value - - :param istream: IStream compatible instance. If its sha is already set - to a value, the object will just be stored in the our database format, + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, in which case the input stream is expected to be in object format ( header + contents ). :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - + def store_async(self, reader): """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as + Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as they are computed. - + :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, + The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. - + :param reader: async.Reader yielding IStream instances. The same instances will be used in the output channel as were received in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how + + **Note:** As some ODB implementations implement this operation atomic, they might + abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) + task = ChannelThreadTask(reader, str(self.store_async), self.store) return pool.add_task(task) - + #} END edit interface - + class FileDBBase(object): - """Provides basic facilities to retrieve files of interest, including + """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" - + def __init__(self, root_path): """Initialize this instance to look for its files at the given root path All subsequent operations will be relative to this path - :raise InvalidDBRoot: + :raise InvalidDBRoot: **Note:** The base will not perform any accessablity checking as the base - might not yet be accessible, but become accessible before the first + might not yet be accessible, but become accessible before the first access.""" super(FileDBBase, self).__init__() self._root_path = root_path - - - #{ Interface + + + #{ Interface def root_path(self): """:return: path at which this db operates""" return self._root_path - + def db_path(self, rela_path): """ - :return: the given relative path relative to our database root, allowing + :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" return join(self._root_path, rela_path) #} END interface - + class CachingDB(object): """A database which uses caches to speed-up access""" - - #{ Interface + + #{ Interface def update_cache(self, force=False): """ Call this method if the underlying data changed to trigger an update of the internal caching structures. - + :param force: if True, the update must be performed. Otherwise the implementation may decide not to perform an update if it thinks nothing has changed. :return: True if an update was performed as something change indeed""" - + # END interface def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed + """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" if isinstance(database, CompoundDB): compounds = list() @@ -209,11 +209,11 @@ def _databases_recursive(database, output): else: output.append(database) # END handle database type - + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): """A database which delegates calls to sub-databases. - + Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" def _set_cache_(self, attr): @@ -223,27 +223,27 @@ def _set_cache_(self, attr): self._db_cache = dict() else: super(CompoundDB, self)._set_cache_(attr) - + def _db_query(self, sha): """:return: database containing the given 20 byte sha :raise BadObject:""" - # most databases use binary representations, prevent converting + # most databases use binary representations, prevent converting # it everytime a database is being queried try: return self._db_cache[sha] except KeyError: pass # END first level cache - + for db in self._dbs: if db.has_object(sha): self._db_cache[sha] = db return db # END for each database raise BadObject(sha) - - #{ ObjectDBR interface - + + #{ ObjectDBR interface + def has_object(self, sha): try: self._db_query(sha) @@ -251,24 +251,24 @@ def has_object(self, sha): except BadObject: return False # END handle exceptions - + def info(self, sha): return self._db_query(sha).info(sha) - + def stream(self, sha): return self._db_query(sha).stream(sha) def size(self): """:return: total size of all contained databases""" return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) - + def sha_iter(self): return chain(*(db.sha_iter() for db in self._dbs)) - + #} END object DBR Interface - + #{ Interface - + def databases(self): """:return: tuple of database instances we use for lookups""" return tuple(self._dbs) @@ -283,7 +283,7 @@ def update_cache(self, force=False): # END if is caching db # END for each database to update return stat - + def partial_to_complete_sha_hex(self, partial_hexsha): """ :return: 20 byte binary sha1 from the given less-than-40 byte hexsha @@ -291,14 +291,14 @@ def partial_to_complete_sha_hex(self, partial_hexsha): :raise AmbiguousObjectName: """ databases = list() _databases_recursive(self, databases) - + len_partial_hexsha = len(partial_hexsha) if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") else: partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - + # END assure successful binary conversion + candidate = None for db in databases: full_bin_sha = None @@ -320,7 +320,5 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if not candidate: raise BadObject(partial_binsha) return candidate - - #} END interface - + #} END interface diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 1d6ad0f26..6e6ec5d1f 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -14,10 +14,11 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName +) + import os __all__ = ('GitDB', ) @@ -30,21 +31,21 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): PackDBCls = PackedDB LooseDBCls = LooseObjectDB ReferenceDBCls = ReferenceDB - + # Directories packs_dir = 'pack' loose_dir = '' alternates_dir = os.path.join('info', 'alternates') - + def __init__(self, root_path): """Initialize ourselves on a git objects directory""" super(GitDB, self).__init__(root_path) - + def _set_cache_(self, attr): if attr == '_dbs' or attr == '_loose_db': self._dbs = list() loose_db = None - for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), (self.loose_dir, self.LooseDBCls), (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) @@ -55,31 +56,30 @@ def _set_cache_(self, attr): # END remember loose db # END check path exists # END for each db type - + # should have at least one subdb if not self._dbs: raise InvalidDBRoot(self.root_path()) # END handle error - + # we the first one should have the store method assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" - + # finally set the value self._loose_db = loose_db else: super(GitDB, self)._set_cache_(attr) # END handle attrs - + #{ ObjectDBW interface - + def store(self, istream): return self._loose_db.store(istream) - + def ostream(self): return self._loose_db.ostream() - + def set_ostream(self, ostream): return self._loose_db.set_ostream(ostream) - + #} END objectdbw interface - diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index dc0ea0e3b..4ebca84d3 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -10,46 +10,46 @@ from gitdb.exc import ( - InvalidDBRoot, + InvalidDBRoot, BadObject, AmbiguousObjectName - ) +) from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, FDStream, Sha1Writer - ) +) from gitdb.base import ( - OStream, - OInfo - ) + OStream, + OInfo +) from gitdb.util import ( - file_contents_ro_filepath, - ENOENT, - hex_to_bin, - bin_to_hex, - exists, - chmod, - isdir, - isfile, - remove, - mkdir, - rename, - dirname, - basename, - join - ) - -from gitdb.fun import ( + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isdir, + isfile, + remove, + mkdir, + rename, + dirname, + basename, + join +) + +from gitdb.fun import ( chunk_size, - loose_object_header_info, + loose_object_header_info, write_object, stream_copy - ) +) import tempfile import mmap @@ -62,11 +62,11 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): """A database which operates on loose object files""" - + # CONFIGURATION # chunks in which data will be copied between streams stream_chunk_size = chunk_size - + # On windows we need to keep it writable, otherwise it cannot be removed # either new_objects_mode = 0444 @@ -81,14 +81,14 @@ def __init__(self, root_path): # Depending on the root, this might work for some mounts, for others not, which # is why it is per instance self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface + + #{ Interface def object_path(self, hexsha): """ - :return: path at which the object with the given hexsha would be stored, + :return: path at which the object with the given hexsha would be stored, relative to the database root""" return join(hexsha[:2], hexsha[2:]) - + def readable_db_object_path(self, hexsha): """ :return: readable object path to the object identified by hexsha @@ -97,8 +97,8 @@ def readable_db_object_path(self, hexsha): return self._hexsha_to_file[hexsha] except KeyError: pass - # END ignore cache misses - + # END ignore cache misses + # try filesystem path = self.db_path(self.object_path(hexsha)) if exists(path): @@ -106,11 +106,11 @@ def readable_db_object_path(self, hexsha): return path # END handle cache raise BadObject(hexsha) - + def partial_to_complete_sha_hex(self, partial_hexsha): """:return: 20 byte binary sha1 string which matches the given name uniquely :param name: hexadecimal partial name - :raise AmbiguousObjectName: + :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for binsha in self.sha_iter(): @@ -123,9 +123,9 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if candidate is None: raise BadObject(partial_hexsha) return candidate - + #} END interface - + def _map_loose_object(self, sha): """ :return: memory map of that file to allow random read access @@ -151,13 +151,13 @@ def _map_loose_object(self, sha): finally: os.close(fd) # END assure file is closed - + def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" if stream is not None and not isinstance(stream, Sha1Writer): raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) return super(LooseObjectDB, self).set_ostream(stream) - + def info(self, sha): m = self._map_loose_object(sha) try: @@ -166,12 +166,12 @@ def info(self, sha): finally: m.close() # END assure release of system resources - + def stream(self, sha): m = self._map_loose_object(sha) type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) return OStream(sha, type, size, stream) - + def has_object(self, sha): try: self.readable_db_object_path(bin_to_hex(sha)) @@ -179,7 +179,7 @@ def has_object(self, sha): except BadObject: return False # END check existance - + def store(self, istream): """note: The sha we produce will be hex by nature""" tmp_path = None @@ -187,14 +187,14 @@ def store(self, istream): if writer is None: # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - + if istream.binsha is None: writer = FDCompressedSha1Writer(fd) else: writer = FDStream(fd) # END handle direct stream copies # END handle custom writer - + try: try: if istream.binsha is not None: @@ -215,14 +215,14 @@ def store(self, istream): os.remove(tmp_path) raise # END assure tmpfile removal on error - + hexsha = None if istream.binsha: hexsha = istream.hexsha else: hexsha = writer.sha(as_hex=True) # END handle sha - + if tmp_path: obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) @@ -234,29 +234,28 @@ def store(self, istream): remove(obj_path) # END handle win322 rename(tmp_path, obj_path) - + # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr chmod(obj_path, self.new_objects_mode) # END handle dry_run - + istream.binsha = hex_to_bin(hexsha) return istream - + def sha_iter(self): # find all files which look like an object, extract sha from there for root, dirs, files in os.walk(self.root_path()): root_base = basename(root) if len(root_base) != 2: continue - + for f in files: if len(f) != 38: continue yield hex_to_bin(root_base + f) # END for each file # END for each walk iteration - + def size(self): return len(tuple(self.sha_iter())) - diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index b9b2b8995..e4fba94b3 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -10,18 +10,14 @@ ) from gitdb.base import ( - OStream, - IStream, - ) + OStream, + IStream, +) from gitdb.exc import ( - BadObject, - UnsupportedOperation - ) -from gitdb.stream import ( - ZippedStoreShaWriter, - DecompressMemMapReader, - ) + BadObject, + UnsupportedOperation +) from cStringIO import StringIO @@ -32,45 +28,45 @@ class MemoryDB(ObjectDBR, ObjectDBW): retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already exists in the target storage before introducing actual IO - + **Note:** memory is currently not threadsafe, hence the async methods cannot be used for storing""" - + def __init__(self): super(MemoryDB, self).__init__() self._db = LooseObjectDB("path/doesnt/matter") - + # maps 20 byte shas to their OStream objects self._cache = dict() - + def set_ostream(self, stream): raise UnsupportedOperation("MemoryDB's always stream into memory") - + def store(self, istream): zstream = ZippedStoreShaWriter() self._db.set_ostream(zstream) - + istream = self._db.store(istream) zstream.close() # close to flush zstream.seek(0) - - # don't provide a size, the stream is written in object format, hence the + + # don't provide a size, the stream is written in object format, hence the # header needs decompression - decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) - + return istream - + def store_async(self, reader): raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - + def has_object(self, sha): return sha in self._cache def info(self, sha): # we always return streams, which are infos as well return self.stream(sha) - + def stream(self, sha): try: ostream = self._cache[sha] @@ -80,15 +76,15 @@ def stream(self, sha): except KeyError: raise BadObject(sha) # END exception handling - + def size(self): return len(self._cache) - + def sha_iter(self): return self._cache.iterkeys() - - - #{ Interface + + + #{ Interface def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly @@ -100,12 +96,12 @@ def stream_copy(self, sha_iter, odb): if odb.has_object(sha): continue # END check object existance - + ostream = self.stream(sha) # compressed data including header sio = StringIO(ostream.stream.data()) istream = IStream(ostream.type, ostream.size, sio, sha) - + odb.store(istream) count += 1 # END for each sha diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 928731937..09f811847 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -12,10 +12,10 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - BadObject, - UnsupportedOperation, - AmbiguousObjectName - ) + BadObject, + UnsupportedOperation, + AmbiguousObjectName +) from gitdb.pack import PackEntity @@ -29,12 +29,12 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): """A database operating on a set of object packs""" - + # sort the priority list every N queries - # Higher values are better, performance tests don't show this has + # Higher values are better, performance tests don't show this has # any effect, but it should have one _sort_interval = 500 - + def __init__(self, root_path): super(PackedDB, self).__init__(root_path) # list of lists with three items: @@ -44,29 +44,29 @@ def __init__(self, root_path): # self._entities = list() # lazy loaded list self._hit_count = 0 # amount of hits self._st_mtime = 0 # last modification data of our root path - + def _set_cache_(self, attr): if attr == '_entities': self._entities = list() self.update_cache(force=True) # END handle entities initialization - + def _sort_entities(self): self._entities.sort(key=lambda l: l[0], reverse=True) - + def _pack_info(self, sha): """:return: tuple(entity, index) for an item at the given sha :param sha: 20 or 40 byte sha :raise BadObject: **Note:** This method is not thread-safe, but may be hit in multi-threaded - operation. The worst thing that can happen though is a counter that + operation. The worst thing that can happen though is a counter that was not incremented, or the list being in wrong order. So we safe the time for locking here, lets see how that goes""" # presort ? if self._hit_count % self._sort_interval == 0: self._sort_entities() # END update sorting - + for item in self._entities: index = item[2](sha) if index is not None: @@ -75,14 +75,14 @@ def _pack_info(self, sha): return (item[1], index) # END index found in pack # END for each item - + # no hit, see whether we have to update packs # NOTE: considering packs don't change very often, we safe this call # and leave it to the super-caller to trigger that raise BadObject(sha) - - #{ Object DB Read - + + #{ Object DB Read + def has_object(self, sha): try: self._pack_info(sha) @@ -90,15 +90,15 @@ def has_object(self, sha): except BadObject: return False # END exception handling - + def info(self, sha): entity, index = self._pack_info(sha) return entity.info_at_index(index) - + def stream(self, sha): entity, index = self._pack_info(sha) return entity.stream_at_index(index) - + def sha_iter(self): sha_list = list() for entity in self.entities(): @@ -108,50 +108,50 @@ def sha_iter(self): yield sha_by_index(index) # END for each index # END for each entity - + def size(self): sizes = [item[1].index().size() for item in self._entities] return reduce(lambda x,y: x+y, sizes, 0) - + #} END object db read - + #{ object db write - + def store(self, istream): - """Storing individual objects is not feasible as a pack is designed to + """Storing individual objects is not feasible as a pack is designed to hold multiple objects. Writing or rewriting packs for single objects is inefficient""" raise UnsupportedOperation() - + def store_async(self, reader): # TODO: add ObjectDBRW before implementing this raise NotImplementedError() - + #} END object db write - - - #{ Interface - + + + #{ Interface + def update_cache(self, force=False): """ - Update our cache with the acutally existing packs on disk. Add new ones, + Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones - + :param force: If True, the cache will be updated even though the directory does not appear to have changed according to its modification timestamp. - :return: True if the packs have been updated so there is new information, + :return: True if the packs have been updated so there is new information, False if there was no change to the pack database""" stat = os.stat(self.root_path()) if not force and stat.st_mtime <= self._st_mtime: return False # END abort early on no change self._st_mtime = stat.st_mtime - + # packs are supposed to be prefixed with pack- by git-convention # get all pack files, figure out what changed pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) our_pack_files = set(item[1].pack().path() for item in self._entities) - + # new packs for pack_file in (pack_files - our_pack_files): # init the hit-counter/priority with the size, a good measure for hit- @@ -159,7 +159,7 @@ def update_cache(self, force=False): entity = PackEntity(pack_file) self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) # END for each new packfile - + # removed packs for pack_file in (our_pack_files - pack_files): del_index = -1 @@ -172,22 +172,22 @@ def update_cache(self, force=False): assert del_index != -1 del(self._entities[del_index]) # END for each removed pack - + # reinitialize prioritiess self._sort_entities() return True - + def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - + def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha - :param partial_binsha: binary sha with less than 20 bytes + :param partial_binsha: binary sha with less than 20 bytes :param canonical_length: length of the corresponding canonical representation. It is required as binary sha's cannot display whether the original hex sha had an odd or even number of characters - :raise AmbiguousObjectName: + :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for item in self._entities: @@ -199,11 +199,11 @@ def partial_to_complete_sha(self, partial_binsha, canonical_length): candidate = sha # END handle full sha could be found # END for each entity - + if candidate: return candidate - + # still not found ? raise BadObject(partial_binsha) - + #} END interface diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 60004a77a..368ab9a61 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -11,16 +11,16 @@ class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" - + # Configuration # Specifies the object database to use for the paths found in the alternates # file. If None, it defaults to the GitDB ObjectDBCls = None - + def __init__(self, ref_file): super(ReferenceDB, self).__init__() self._ref_file = ref_file - + def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() @@ -28,7 +28,7 @@ def _set_cache_(self, attr): else: super(ReferenceDB, self)._set_cache_(attr) # END handle attrs - + def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls if dbcls is None: @@ -36,7 +36,7 @@ def _update_dbs_from_ref_file(self): from git import GitDB dbcls = GitDB # END get db type - + # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: @@ -44,10 +44,10 @@ def _update_dbs_from_ref_file(self): except (OSError, IOError): pass # END handle alternates - + ref_paths_set = set(ref_paths) cur_ref_paths_set = set(db.root_path() for db in self._dbs) - + # remove existing for path in (cur_ref_paths_set - ref_paths_set): for i, db in enumerate(self._dbs[:]): @@ -56,7 +56,7 @@ def _update_dbs_from_ref_file(self): continue # END del matching db # END for each path to remove - + # add new # sort them to maintain order added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) @@ -72,7 +72,7 @@ def _update_dbs_from_ref_file(self): # ignore invalid paths or issues pass # END for each path to add - + def update_cache(self, force=False): # re-read alternates and update databases self._update_dbs_from_ref_file() diff --git a/gitdb/exc.py b/gitdb/exc.py index 7180fb586..47fc80912 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -7,17 +7,17 @@ class ODBError(Exception): """All errors thrown by the object database""" - + class InvalidDBRoot(ODBError): """Thrown if an object database cannot be initialized at the given path""" - + class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the + """The object with the given SHA does not exist. Instantiate with the failed sha""" - + def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) - + class ParseError(ODBError): """Thrown if the parsing of a file failed due to an invalid format""" diff --git a/gitdb/fun.py b/gitdb/fun.py index c1e73e895..9e5c44a6c 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -24,30 +24,30 @@ delta_types = (OFS_DELTA, REF_DELTA) type_id_to_type_map = { - 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", - 5 : "", # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA - } + 0 : "", # EXT 1 + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA +} type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA - ) + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA +) # used when dealing with larger streams -chunk_size = 1000*mmap.PAGESIZE +chunk_size = 1000 * mmap.PAGESIZE -__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') @@ -59,11 +59,11 @@ def _set_delta_rbound(d, size): to our size :return: d""" d.ts = size - + # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE return d - + def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static @@ -71,19 +71,19 @@ def _move_delta_lbound(d, bytes): :return: d""" if bytes == 0: return - + d.to += bytes d.so += bytes d.ts -= bytes if d.data is not None: d.data = d.data[bytes:] # END handle data - + return d - + def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) - + def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer :param bbuf: buffer providing source bytes for copy operations @@ -107,13 +107,13 @@ class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" __slots__ = ( - 'to', # start offset in the target buffer in bytes + 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None ) - + def __init__(self, to, ts, so, data): self.to = to self.ts = ts @@ -122,22 +122,22 @@ def __init__(self, to, ts, so, data): def __repr__(self): return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") - + #{ Interface - + def rbound(self): return self.to + self.ts - + def has_data(self): """:return: True if the instance has data to add to the target stream""" return self.data is not None - + #} END interface def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs. + absofs. **Note:** global method for performance only, it belongs to DeltaChunkList""" lo = 0 hi = len(dcl) @@ -153,7 +153,7 @@ def _closest_index(dcl, absofs): # END handle bound # END for each delta absofs return len(dcl)-1 - + def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed write function. @@ -166,14 +166,14 @@ def delta_list_apply(dcl, bbuf, write): # END for each dc def delta_list_slice(dcl, absofs, size, ndcl): - """:return: Subsection of this list at the given absolute offset, with the given + """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: None""" cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - lappend = ndcl.append - + lappend = ndcl.append + if cd.to != absofs: tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) _move_delta_lbound(tcd, absofs - cd.to) @@ -182,7 +182,7 @@ def delta_list_slice(dcl, absofs, size, ndcl): size -= tcd.ts cdi += 1 # END lbound overlap handling - + while cdi < slen and size: # are we larger than the current block cd = dcl[cdi] @@ -198,38 +198,38 @@ def delta_list_slice(dcl, absofs, size, ndcl): # END hadle size cdi += 1 # END for each chunk - - + + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. There are two types of lists we represent. The one was created bottom-up, working - towards the latest delta, the other kind was created top-down, working from the - latest delta down to the earliest ancestor. This attribute is queryable + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable after all processing with is_reversed.""" - + __slots__ = tuple() - + def rbound(self): """:return: rightmost extend in bytes, absolute""" if len(self) == 0: return 0 return self[-1].rbound() - + def lbound(self): """:return: leftmost byte at which this chunklist starts""" if len(self) == 0: return 0 return self[0].to - + def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - + def apply(self, bbuf, write): """Only used by public clients, internally we only use the global routines for performance""" return delta_list_apply(self, bbuf, write) - + def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate add-chunks @@ -239,7 +239,7 @@ def compress(self): return self i = 0 slen_orig = slen - + first_data_index = None while i < slen: dc = self[i] @@ -253,27 +253,27 @@ def compress(self): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data - + del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) - + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + slen = len(self) i = first_data_index + 1 - + # END concatenate data first_data_index = None continue # END skip non-data chunks - + if first_data_index is None: first_data_index = i-1 # END iterate list - + #if slen_orig != len(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self - + def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -283,39 +283,39 @@ def check_integrity(self, target_size=-1): assert self[-1].rbound() == target_size assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size # END target size verification - + if len(self) < 2: return - + # check data for dc in self: assert dc.ts > 0 if dc.has_data(): assert len(dc.data) >= dc.ts # END for each dc - + left = islice(self, 0, len(self)-1) right = iter(self) right.next() - # this is very pythonic - we might have just use index based access here, + # this is very pythonic - we might have just use index based access here, # but this could actually be faster for lft,rgt in izip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair - + class TopdownDeltaChunkList(DeltaChunkList): - """Represents a list which is generated by feeding its ancestor streams one by + """Represents a list which is generated by feeding its ancestor streams one by one""" - __slots__ = tuple() - + __slots__ = tuple() + def connect_with_next_base(self, bdcl): """Connect this chain with the next level of our base delta chunklist. The goal in this game is to mark as many of our chunks rigid, hence they - cannot be changed by any of the upcoming bases anymore. Once all our + cannot be changed by any of the upcoming bases anymore. Once all our chunks are marked like that, we can stop all processing - :param bdcl: data chunk list being one of our bases. They must be fed in + :param bdcl: data chunk list being one of our bases. They must be fed in consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams if False is returned""" @@ -326,13 +326,13 @@ def connect_with_next_base(self, bdcl): while dci < slen: dc = self[dci] dci += 1 - + # all add-chunks which are already topmost don't need additional processing if dc.data is not None: nfc += 1 continue # END skip add chunks - + # copy chunks # integrate the portion of the base list into ourselves. Lists # dont support efficient insertion ( just one at a time ), but for now @@ -341,13 +341,13 @@ def connect_with_next_base(self, bdcl): # ourselves in order to reduce the amount of insertions ... del(ccl[:]) delta_list_slice(bdcl, dc.so, dc.ts, ccl) - + # move the target bounds into place to match with our chunk ofs = dc.to - dc.so for cdc in ccl: cdc.to += ofs # END update target bounds - + if len(ccl) == 1: self[dci-1] = ccl[0] else: @@ -359,19 +359,19 @@ def connect_with_next_base(self, bdcl): del(self[dci-1:]) # include deletion of dc self.extend(ccl) self.extend(post_dci) - + slen = len(self) dci += len(ccl)-1 # deleted dc, added rest - + # END handle chunk replacement # END for each chunk - + if nfc == slen: return False # END handle completeness return True - - + + #} END structures #{ Routines @@ -386,14 +386,14 @@ def is_loose_object(m): def loose_object_header_info(m): """ - :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find("\0")].split(" ") return type_name, int(size) - + def pack_object_header_info(data): """ :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) @@ -417,7 +417,7 @@ def create_pack_object_header(obj_type, obj_size): """ :return: string defining the pack header comprised of the object type and its incompressed size in bytes - + :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte @@ -432,10 +432,10 @@ def create_pack_object_header(obj_type, obj_size): #END until size is consumed hdr += chr(c) return hdr - + def msb_size(data, offset=0): """ - :return: tuple(read_bytes, size) read the msb size from the given random + :return: tuple(read_bytes, size) read the msb size from the given random access data starting at the given byte offset""" size = 0 i = 0 @@ -452,19 +452,19 @@ def msb_size(data, offset=0): # END while in range if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size - + return i+offset, size + def loose_object_header(type, size): """ :return: string representing the loose object header, which is immediately followed by the content stream of size 'size'""" return "%s %i\0" % (type, size) - + def write_object(type, size, read, write, chunk_size=chunk_size): """ - Write the object as identified by type, size and source_stream into the + Write the object as identified by type, size and source_stream into the target_stream - + :param type: type string of the object :param size: amount of bytes to write from source_stream :param read: read method of a stream providing the content data @@ -473,26 +473,26 @@ def write_object(type, size, read, write, chunk_size=chunk_size): the routine exits, even if an error is thrown :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" tbw = 0 # total num bytes written - + # WRITE HEADER: type SP size NULL tbw += write(loose_object_header(type, size)) tbw += stream_copy(read, write, size, chunk_size) - + return tbw def stream_copy(read, write, size, chunk_size): """ - Copy a stream up to size bytes using the provided read and write methods, + Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size - + **Note:** its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written - + # WRITE ALL DATA UP TO SIZE while True: cs = min(chunk_size, size-dbw) # NOTE: not all write methods return the amount of written bytes, like - # mmap.write. Its bad, but we just deal with it ... perhaps its not + # mmap.write. Its bad, but we just deal with it ... perhaps its not # even less efficient # data_len = write(read(cs)) # dbw += data_len @@ -505,27 +505,27 @@ def stream_copy(read, write, size, chunk_size): # END check for stream end # END duplicate data return dbw - + def connect_deltas(dstreams): """ Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - + :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" tdcl = None # topmost dcl - + dcl = tdcl = TopdownDeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() delta_buf_size = ds.size - + # read header i, base_size = msb_size(db) i, target_size = msb_size(db, i) - + # interpret opcodes tbw = 0 # amount of target bytes written while i < delta_buf_size: @@ -554,15 +554,15 @@ def connect_deltas(dstreams): if (c & 0x40): cp_size |= (ord(db[i]) << 16) i += 1 - - if not cp_size: + + if not cp_size: cp_size = 0x10000 - + rbound = cp_off + cp_size if (rbound < cp_size or rbound > base_size): break - + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: @@ -575,31 +575,31 @@ def connect_deltas(dstreams): raise ValueError("unexpected delta opcode 0") # END handle command byte # END while processing delta data - + dcl.compress() - + # merge the lists ! if dsi > 0: if not tdcl.connect_with_next_base(dcl): break # END handle merge - + # prepare next base dcl = DeltaChunkList() # END for each delta stream - + return tdcl - + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file - + :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes - + **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf @@ -629,10 +629,10 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (c & 0x40): cp_size |= (ord(db[i]) << 16) i += 1 - - if not cp_size: + + if not cp_size: cp_size = 0x10000 - + rbound = cp_off + cp_size if (rbound < cp_size or rbound > src_buf_size): @@ -645,28 +645,28 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): raise ValueError("unexpected delta opcode 0") # END handle command byte # END while processing delta data - + # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" - - + + def is_equal_canonical_sha(canonical_length, match, sha1): """ :return: True if the given lhs and rhs 20 byte binary shas - The comparison will take the canonical_length of the match sha into account, + The comparison will take the canonical_length of the match sha into account, hence the comparison will only use the last 4 bytes for uneven canonical representations :param match: less than 20 byte sha :param sha1: 20 byte sha""" binary_length = canonical_length/2 if match[:binary_length] != sha1[:binary_length]: return False - + if canonical_length - binary_length and \ (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: return False # END handle uneven canonnical length return True - + #} END routines diff --git a/gitdb/pack.py b/gitdb/pack.py index 48121f026..4a0badccb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -54,9 +54,9 @@ ) from struct import ( - pack, - unpack, - ) + pack, + unpack, +) from binascii import crc32 @@ -68,10 +68,10 @@ __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') - - -#{ Utilities + + +#{ Utilities def pack_object_at(cursor, offset, as_stream): """ @@ -81,13 +81,13 @@ def pack_object_at(cursor, offset, as_stream): data to be read decompressed. :param data: random accessable data containing all required information :parma offset: offset in to the data at which the object information is located - :param as_stream: if True, a stream object will be returned that can read + :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None - + # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset @@ -111,7 +111,7 @@ def pack_object_at(cursor, offset, as_stream): # assume its a base object total_rela_offset = data_rela_offset # END handle type id - + abs_data_offset = offset + total_rela_offset if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) @@ -141,29 +141,29 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): if want_crc: crc = base_crc #END initialize crc - + while True: chunk = read(chunk_size) br += len(chunk) compressed = zstream.compress(chunk) bw += len(compressed) write(compressed) # cannot assume return value - + if want_crc: crc = crc32(compressed, crc) #END handle crc - + if len(chunk) != chunk_size: break #END copy loop - + compressed = zstream.flush() bw += len(compressed) write(compressed) if want_crc: crc = crc32(compressed, crc) #END handle crc - + return (br, bw, crc) @@ -175,26 +175,26 @@ class IndexWriter(object): in one go to the given stream **Note:** currently only writes v2 indices""" __slots__ = '_objs' - + def __init__(self): self._objs = list() - + def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - + def write(self, pack_sha, write): """Write the index file using the given write method :param pack_sha: binary sha over the whole pack that we index :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) - + sha_writer = FlexibleSha1Writer(write) sha_write = sha_writer.write sha_write(PackIndexFile.index_v2_signature) sha_write(pack(">L", PackIndexFile.index_version_default)) - + # fanout tmplist = list((0,)*256) # fanout or list with 64 bit offsets for t in self._objs: @@ -206,16 +206,16 @@ def write(self, pack_sha, write): tmplist[i+1] += v #END write each fanout entry sha_write(pack('>L', tmplist[255])) - + # sha1 ordered # save calls, that is push them into c sha_write(''.join(t[0] for t in self._objs)) - + # crc32 for t in self._objs: sha_write(pack('>L', t[1]&0xffffffff)) #END for each crc - + tmplist = list() # offset 32 for t in self._objs: @@ -226,28 +226,28 @@ def write(self, pack_sha, write): #END hande 64 bit offsets sha_write(pack('>L', ofs&0xffffffff)) #END for each offset - + # offset 64 for ofs in tmplist: sha_write(pack(">Q", ofs)) #END for each offset - + # trailer assert(len(pack_sha) == 20) sha_write(pack_sha) sha = sha_writer.sha(as_hex=False) write(sha) return sha - - + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" - + # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -258,7 +258,7 @@ class PackIndexFile(LazyMixin): def __init__(self, indexpath): super(PackIndexFile, self).__init__() self._indexpath = indexpath - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -276,91 +276,91 @@ def _set_cache_(self, attr): else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties - + # CHECK VERSION mmap = self._cursor.map() self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version - + # SETUP FUNCTIONS # setup our functions according to the actual version for fname in ('entry', 'offset', 'sha', 'crc'): setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) # END for each function to initialize - - + + # INITIALIZE DATA # byte offset is 8 if version is 2, 0 otherwise self._initialize() # END handle attributes - + #{ Access V1 - + def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + def _offset_v1(self, i): """see ``_offset_v2``""" return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - + def _sha_v1(self, i): """see ``_sha_v2``""" base = 1024 + (i*24)+4 return self._cursor.map()[base:base+20] - + def _crc_v1(self, i): """unsupported""" return 0 - + #} END access V1 - + #{ Access V2 def _entry_v2(self, i): """:return: tuple(offset, binsha, crc)""" return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - + def _offset_v2(self, i): - """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - + # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset - + return offset - + def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 return self._cursor.map()[base:base+20] - + def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + #} END access V2 - + #{ Initialization - + def _initialize(self): """initialize base data""" self._fanout_table = self._read_fanout((self._version == 2) * 8) - + if self._version == 2: self._crc_list_offset = self._sha_list_offset + self.size() * 20 self._pack_offset = self._crc_list_offset + self.size() * 4 self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base - + def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" d = self._cursor.map() @@ -370,38 +370,38 @@ def _read_fanout(self, byte_offset): append(unpack_from('>L', d, byte_offset + i*4)[0]) # END for each entry return out - + #} END initialization - + #{ Properties def version(self): return self._version - + def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - + def path(self): """:return: path to the packindexfile""" return self._indexpath - + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._cursor.map()[-40:-20] - + def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._cursor.map()[-20:] - + def offsets(self): """:return: sequence of all offsets in the order in which they were written - + **Note:** return value can be random accessed, but may be immmutable""" if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - + # networkbyteorder to something array likes more if sys.byteorder == 'little': a.byteswap() @@ -409,7 +409,7 @@ def offsets(self): else: return tuple(self.offset(index) for index in xrange(self.size())) # END handle version - + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -421,7 +421,7 @@ def sha_to_index(self, sha): if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 @@ -435,29 +435,29 @@ def sha_to_index(self, sha): # END handle midpoint # END bisect return None - + def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this index file :param partial_bin_sha: an at least two bytes of a partial binary sha - :param canonical_length: lenght of the original hexadecimal representation of the + :param canonical_length: lenght of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # fill the partial to full 20 bytes filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest + + # find lowest while lo < hi: mid = (lo + hi) / 2 c = cmp(filled_sha, get_sha(mid)) @@ -471,7 +471,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): lo = mid + 1 # END handle midpoint # END bisect - + if lo < self.size(): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): @@ -484,86 +484,86 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): # END if we have a match # END if we found something return None - + if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses def sha_to_index(self, sha): return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - + # END redefine heavy-hitter with c version + #} END properties - - + + class PackFile(LazyMixin): """A pack is a file written according to the Version 2 for git packs - + As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - - **Note:** at some point, this might be implemented using streams as well, or + + **Note:** at some point, this might be implemented using streams as well, or streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that + for some reason - one clearly doesn't want to read 10GB at once in that case""" - + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 - + # offset into our data at which the first object starts first_object_offset = 3*4 # header bytes footer_size = 20 # final sha - + def __init__(self, packpath): self._packpath = packpath - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() - + # read the header information type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - + # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" c = self._cursor content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset - + null = NullStream() while cur_offset < content_size: data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset - + stream_copy(ostream.read, null.write, ostream.size, chunk_size) cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - + + # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the + # Otherwise return the Stream object directly, its derived from the # info object if as_stream: ostream.stream.seek(0) yield ostream # END until we have read everything - + #{ Pack Information - + def size(self): - """:return: The amount of objects stored in this pack""" + """:return: The amount of objects stored in this pack""" return self._size - + def version(self): """:return: the version of this pack""" return self._version - + def data(self): """ :return: read-only data of this pack. It provides random access and usually @@ -571,22 +571,22 @@ def data(self): :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" # can use map as we are starting at offset 0. Otherwise we would have to use buffer() return self._cursor.use_region().map() - + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - + def path(self): """:return: path to the packfile""" return self._packpath #} END pack information - + #{ Pack Specific - + def collect_streams(self, offset): """ :return: list of pack streams which are required to build the object - at the given offset. The first entry of the list is the object at offset, + at the given offset. The first entry of the list is the object at offset, the last one is either a full object, or a REF_Delta stream. The latter type needs its reference object to be locked up in an ODB to form a valid delta chain. @@ -601,7 +601,7 @@ def collect_streams(self, offset): offset = ostream.pack_offset - ostream.delta_info else: # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter + # else is either an object, or a ref delta, in the latter # case someone else has to find it break # END handle type @@ -609,55 +609,55 @@ def collect_streams(self, offset): return out #} END pack specific - + #{ Read-Database like Interface - + def info(self, offset): """Retrieve information about the object at the given file-absolute offset - + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - + def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information - + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - + def stream_iter(self, start_offset=0): """ - :return: iterator yielding OPackStream compatible instances, allowing + :return: iterator yielding OPackStream compatible instances, allowing to access the data in the pack directly. - :param start_offset: offset to the first object to iterate. If 0, iteration + :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. - + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) - + #} END Read-Database like Interface - - + + class PackEntity(LazyMixin): - """Combines the PackIndexFile and the PackFile into one, allowing the + """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file + + __slots__ = ( '_index', # our index file '_pack', # our pack file '_offset_map' # on demand dict mapping one offset to the next consecutive one ) - + IndexFileCls = PackIndexFile PackFileCls = PackFile - + def __init__(self, pack_or_index_path): """Initialize ourselves with the path to the respective pack or index file""" basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - + def _set_cache_(self, attr): # currently this can only be _offset_map # TODO: make this a simple sorted offset array which can be bisected @@ -666,7 +666,7 @@ def _set_cache_(self, attr): offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" - + offset_map = None if len(offsets_sorted) == 1: offset_map = { offsets_sorted[0] : last_offset } @@ -675,21 +675,21 @@ def _set_cache_(self, attr): iter_offsets_plus_one = iter(offsets_sorted) iter_offsets_plus_one.next() consecutive = izip(iter_offsets, iter_offsets_plus_one) - + offset_map = dict(consecutive) - + # the last offset is not yet set offset_map[offsets_sorted[-1]] = last_offset # END handle offset amount self._offset_map = offset_map - + def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) if index is None: raise BadObject(sha) return index - + def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha @@ -697,7 +697,7 @@ def _iter_objects(self, as_stream): for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index - + def _object(self, sha, as_stream, index=-1): """:return: OInfo or OStream object providing information about the given sha :param index: if not -1, its assumed to be the sha's index in the IndexFile""" @@ -714,86 +714,86 @@ def _object(self, sha, as_stream, index=-1): packstream = self._pack.stream(offset) return OStream(sha, packstream.type, packstream.size, packstream.stream) # END handle non-deltas - + # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, + # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) + + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) # END handle non-deltas - + # deltas are a little tougher - unpack the first bytes to obtain # the actual target size, as opposed to the size of the delta data streams = self.collect_streams_at_offset(offset) buf = streams[0].read(512) offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - + # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) + return OInfo(sha, streams[-1].type, target_size) # END handle stream - + #{ Read-Database like Interface - + def info(self, sha): """Retrieve information about the object identified by the given sha - + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" return self._object(sha, False) - + def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha - + :param sha: 20 byte sha1 - :raise BadObject: + :raise BadObject: :return: OStream instance, with 20 byte sha""" return self._object(sha, True) def info_at_index(self, index): """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, False, index) - + def stream_at_index(self, index): - """As ``stream``, but uses a PackIndexFile compatible index to refer to the + """As ``stream``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, True, index) - + #} END Read-Database like Interface - - #{ Interface + + #{ Interface def pack(self): """:return: the underlying pack file instance""" return self._pack - + def index(self): """:return: the underlying pack index file instance""" return self._index - + def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - - :param use_crc: if True, the index' crc is run over the compressed stream of + + :param use_crc: if True, the index' crc is run over the compressed stream of the object, which is much faster than checking the sha1. It is also more prone to unnoticed corruption or manipulation. :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is - a delta, this only verifies that the delta's data is valid, not the - data of the actual undeltified object, as it depends on more than + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha - + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -801,12 +801,12 @@ def is_valid_stream(self, sha, use_crc=False): if self._index.version() < 2: raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") # END handle index version - + index = self._sha_to_index(sha) offset = self._index.offset(index) next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - + # create the current crc value, on the compressed object data # Read it in chunks, without copying the data crc_update = zlib.crc32 @@ -819,7 +819,7 @@ def is_valid_stream(self, sha, use_crc=False): this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - + # crc returns signed 32 bit numbers, the AND op forces it into unsigned # mode ... wow, sneaky, from dulwich. return (this_crc_value & 0xffffffff) == crc_value @@ -828,7 +828,7 @@ def is_valid_stream(self, sha, use_crc=False): stream = self._object(sha, as_stream=True) # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) - + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification @@ -839,21 +839,21 @@ def info_iter(self): :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) - + def stream_iter(self): """ :return: iterator over all objects in this pack. The iterator yields OStream instances""" return self._iter_objects(as_stream=True) - + def collect_streams_at_offset(self, offset): """ As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` - + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) - + # try to resolve the last one if needed. It is assumed to be either # a REF delta, or a base object, as OFFSET deltas are resolved by the pack if streams[-1].type_id == REF_DELTA: @@ -866,54 +866,54 @@ def collect_streams_at_offset(self, offset): stream = self._pack.stream(self._index.offset(sindex)) streams.append(stream) else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who # would do that ;) ? We can handle it though stream = self._pack.stream(stream.delta_info) streams.append(stream) # END handle ref delta # END resolve ref streams # END resolve streams - + return streams - + def collect_streams(self, sha): """ As ``PackFile.collect_streams``, but takes a sha instead of an offset. Additionally, ref_delta streams will be resolved within this pack. If this is not possible, the stream will be left alone, hence it is adivsed - to check for unresolved ref-deltas and resolve them before attempting to + to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. - + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect - :return: list of streams, first being the actual object delta, the last being + :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. :raise BadObject:""" return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - + + @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, + def write_pack(cls, object_iter, pack_write, index_write=None, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. The respective index is produced as well if index_write is not Non. - + :param object_iter: iterator yielding odb output objects :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding to the pack. - :param object_count: if you can provide the amount of objects in your iteration, - this would be the place to put it. Otherwise we have to pre-iterate and store + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store all items into a list to get the number, which uses more memory than necessary. :param zlib_compression: the zlib compression level to use :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack and over all contents of the index. If index_write was None, index_binsha will be None - + **Note:** The destination of the write functions is up to the user. It could be a socket, or a file for instance - + **Note:** writes only undeltified objects""" objs = object_iter if not object_count: @@ -922,26 +922,26 @@ def write_pack(cls, object_iter, pack_write, index_write=None, #END handle list type object_count = len(objs) #END handle object - + pack_writer = FlexibleSha1Writer(pack_write) pwrite = pack_writer.write ofs = 0 # current offset into the pack file index = None wants_index = index_write is not None - + # write header pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) ofs += 12 - + if wants_index: index = IndexWriter() #END handle index header - + actual_count = 0 for obj in objs: actual_count += 1 crc = 0 - + # object header hdr = create_pack_object_header(obj.type_id, obj.size) if index_write: @@ -950,7 +950,7 @@ def write_pack(cls, object_iter, pack_write, index_write=None, crc = None #END handle crc pwrite(hdr) - + # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream @@ -959,54 +959,54 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if wants_index: index.append(obj.binsha, crc, ofs) #END handle index - + ofs += len(hdr) + bw if actual_count == object_count: break #END abort once we are done #END for each object - + if actual_count != object_count: raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) #END count assertion - + # write footer pack_sha = pack_writer.sha(as_hex = False) assert len(pack_sha) == 20 pack_write(pack_sha) ofs += len(pack_sha) # just for completeness ;) - + index_sha = None if wants_index: index_sha = index.write(pack_sha, index_write) #END handle index - + return pack_sha, index_sha - + @classmethod def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """Create a new on-disk entity comprised of a properly named pack file and a properly named and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files :return: PackEntity instance initialized with the new pack - + **Note:** for more information on the other parameters see the write_pack method""" pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) index_write = lambda d: os.write(index_fd, d) - + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) os.close(pack_fd) os.close(index_fd) - + fmt = "pack-%s.%s" new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) os.rename(pack_path, new_pack_path) os.rename(index_path, new_index_path) - + return cls(new_pack_path) - - + + #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 6441b1e1a..b21c39c9d 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,7 @@ except ImportError: pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', 'FDStream', 'NullStream') @@ -41,27 +41,27 @@ #{ RO Streams class DecompressMemMapReader(LazyMixin): - """Reads data in chunks from a memory map and decompresses it. The client sees + """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly - - A constraint on the total size of bytes is activated, simulating + + A constraint on the total size of bytes is activated, simulating a logical file within a possibly larger physical memory area - - To read efficiently, you clearly don't want to read individual bytes, instead, + + To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. - - **Note:** The chunk-size should be carefully selected as it will involve quite a bit - of string copying due to the way the zlib is implemented. Its very wasteful, - hence we try to find a good tradeoff between allocation time and number of + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', '_cbr', '_phi') - + max_read_size = 512*1024 # currently unused - + def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no size""" @@ -77,22 +77,22 @@ def __init__(self, m, close_on_deletion, size=None): self._cbr = 0 # number of compressed bytes read self._phi = False # is True if we parsed the header info self._close = close_on_deletion # close the memmap on deletion ? - + def _set_cache_(self, attr): assert attr == '_s' - # only happens for size, which is a marker to indicate we still + # only happens for size, which is a marker to indicate we still # have to parse the header from the stream self._parse_header_info() - + def __del__(self): if self._close: self._m.close() # END handle resource freeing - + def _parse_header_info(self): - """If this stream contains object data, parse the header info and skip the + """If this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content - + :return: parsed type_string, size""" # read header maxb = 512 # should really be enough, cgit uses 8192 I believe @@ -102,28 +102,28 @@ def _parse_header_info(self): type, size = hdr[:hdrend].split(" ") size = int(size) self._s = size - + # adjust internal state to match actual header length that we ignore # The buffer will be depleted first on future reads self._br = 0 hdrend += 1 # count terminating \0 self._buf = StringIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend - + self._phi = True - + return type, size - - #{ Interface - + + #{ Interface + @classmethod def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream - This method parses the object header from m and returns the parsed + This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. - + :param m: memory map on which to oparate. It must be object data ( header + contents ) - :param close_on_deletion: if True, the memory map will be closed once we are + :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) type, size = inst._parse_header_info() @@ -131,30 +131,30 @@ def new(self, m, close_on_deletion=False): def data(self): """:return: random access compatible data we are working on""" - return self._m - + return self._m + def compressed_bytes_read(self): """ - :return: number of compressed bytes read. This includes the bytes it + :return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )""" # ABSTRACT: When decompressing a byte stream, it can be that the first - # x bytes which were requested match the first x bytes in the loosely + # x bytes which were requested match the first x bytes in the loosely # compressed datastream. This is the worst-case assumption that the reader # does, it assumes that it will get at least X bytes from X compressed bytes # in call cases. - # The caveat is that the object, according to our known uncompressed size, + # The caveat is that the object, according to our known uncompressed size, # is already complete, but there are still some bytes left in the compressed # stream that contribute to the amount of compressed bytes. # How can we know that we are truly done, and have read all bytes we need - # to read ? - # Without help, we cannot know, as we need to obtain the status of the + # to read ? + # Without help, we cannot know, as we need to obtain the status of the # decompression. If it is not finished, we need to decompress more data # until it is finished, to yield the actual number of compressed bytes # belonging to the decompressed object - # We are using a custom zlib module for this, if its not present, + # We are using a custom zlib module for this, if its not present, # we try to put in additional bytes up for decompression if feasible # and check for the unused_data. - + # Only scrub the stream forward if we are officially done with the # bytes we were to have. if self._br == self._s and not self._zip.unused_data: @@ -171,45 +171,45 @@ def compressed_bytes_read(self): self.read(mmap.PAGESIZE) # END scrub-loop default zlib # END handle stream scrubbing - + # reset bytes read, just to be sure self._br = self._s # END handle stream scrubbing - + # unused data ends up in the unconsumed tail, which was removed # from the count already return self._cbr - - #} END interface - + + #} END interface + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset - + self._zip = zlib.decompressobj() self._br = self._cws = self._cwe = self._cbr = 0 if self._phi: self._phi = False del(self._s) # trigger header parsing on first access # END skip header - + def read(self, size=-1): if size < 1: size = self._s - self._br else: size = min(size, self._s - self._br) # END clamp size - + if size == 0: return str() # END handle depletion - - - # deplete the buffer, then just continue using the decompress object - # which has an own buffer. We just need this to transparently parse the + + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the # header from the zlib stream dat = str() if self._buf: @@ -223,26 +223,26 @@ def read(self, size=-1): dat = self._buf.read() # ouch, duplicates data size -= self._buflen self._br += self._buflen - + self._buflen = 0 self._buf = None # END handle buffer len # END handle buffer - + # decompress some data - # Abstract: zlib needs to operate on chunks of our memory map ( which may + # Abstract: zlib needs to operate on chunks of our memory map ( which may # be large ), as it will otherwise and always fill in the 'unconsumed_tail' - # attribute which possible reads our whole map to the end, forcing + # attribute which possible reads our whole map to the end, forcing # everything to be read from disk even though just a portion was requested. - # As this would be a nogo, we workaround it by passing only chunks of data, - # moving the window into the memory map along as we decompress, which keeps + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps # the tail smaller than our chunk-size. This causes 'only' the chunk to be # copied once, and another copy of a part of it when it creates the unconsumed # tail. We have to use it to hand in the appropriate amount of bytes durin g # the next read. tail = self._zip.unconsumed_tail if tail: - # move the window, make it as large as size demands. For code-clarity, + # move the window, make it as large as size demands. For code-clarity, # we just take the chunk from our map again instead of reusing the unconsumed # tail. The latter one would safe some memory copying, but we could end up # with not getting enough data uncompressed, so we had to sort that out as well. @@ -253,18 +253,18 @@ def read(self, size=-1): else: cws = self._cws self._cws = self._cwe - self._cwe = cws + size + self._cwe = cws + size # END handle tail - - + + # if window is too small, make it larger so zip can decompress something if self._cwe - self._cws < 8: self._cwe = self._cws + 8 # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... + + # takes a slice, but doesn't copy the data, it says ... indata = buffer(self._m, self._cws, self._cwe - self._cws) - + # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) dcompdat = self._zip.decompress(indata, size) @@ -274,13 +274,13 @@ def read(self, size=-1): # if we hit the end of the stream self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) - + if dat: dcompdat = dat + dcompdat # END prepend our cached data - - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. + + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. # Recursively resolve that. # Note: dcompdat can be empty even though we still appear to have bytes # to read, if we are called by compressed_bytes_read - it manipulates @@ -290,30 +290,30 @@ def read(self, size=-1): # END handle special case return dcompdat - + class DeltaApplyReader(LazyMixin): - """A reader which dynamically applies pack deltas to a base object, keeping the + """A reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. - - The size of the final object is only obtainable once all deltas have been + + The size of the final object is only obtainable once all deltas have been applied, unless it is retrieved from a pack index. - + The uncompressed Delta has the following layout (MSB being a most significant bit encoded dynamic size): - + * MSB Source Size - the size of the base against which the delta was created * MSB Target Size - the size of the resulting data after the delta was applied * A list of one byte commands (cmd) which are followed by a specific protocol: - + * cmd & 0x80 - copy delta_data[offset:offset+size] - + * Followed by an encoded offset into the delta data * Followed by an encoded size of the chunk to copy - + * cmd & 0x7f - insert - + * insert cmd bytes from the delta buffer into the output stream - + * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( @@ -321,38 +321,38 @@ class DeltaApplyReader(LazyMixin): "_dstreams", # tuple of delta stream readers "_mm_target", # memory map of the delta-applied data "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read + "_br" # number of bytes read ) - + #{ Configuration k_max_memory_move = 250*1000*1000 #} END configuration - + def __init__(self, stream_list): - """Initialize this instance with a list of streams, the first stream being + """Initialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the base object onto which to apply the deltas""" assert len(stream_list) > 1, "Need at least one delta and one base stream" - + self._bstream = stream_list[-1] self._dstreams = tuple(stream_list[:-1]) self._br = 0 - + def _set_cache_too_slow_without_c(self, attr): - # the direct algorithm is fastest and most direct if there is only one + # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python, every function call costs + # than X - definitely the case in python, every function call costs # huge amounts of time # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) - - # Aggregate all deltas into one delta in reverse order. Hence we take + + # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) - + # call len directly, as the (optional) c version doesn't implement the sequence # protocol if dcl.rbound() == 0: @@ -360,22 +360,22 @@ def _set_cache_too_slow_without_c(self, attr): self._mm_target = allocate_memory(0) return # END handle empty list - + self._size = dcl.rbound() self._mm_target = allocate_memory(self._size) - + bbuf = allocate_memory(self._bstream.size) stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - + # APPLY CHUNKS write = self._mm_target.write dcl.apply(bbuf, write) - + self._mm_target.seek(0) - + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" - + # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the # delta is not peaked into, causing less overhead. @@ -388,37 +388,37 @@ def _set_cache_brute_(self, attr): buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream - + # sanity check - the first delta to apply should have the same source # size as our actual base stream base_size = self._bstream.size target_size = max_target_size - + # if we have more than 1 delta to apply, we will swap buffers, hence we must # assure that all buffers we use are large enough to hold all the results if len(self._dstreams) > 1: base_size = target_size = max(base_size, max_target_size) # END adjust buffer sizes - - + + # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) - + # allocate memory map large enough for the largest (intermediate) target - # We will use it as scratch space for all delta ops. If the final + # We will use it as scratch space for all delta ops. If the final # target buffer is smaller than our allocated space, we just use parts # of it upon return. tbuf = allocate_memory(target_size) - - # for each delta to apply, memory map the decompressed delta and + + # for each delta to apply, memory map the decompressed delta and # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. final_target_size = None for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): - # allocate a buffer to hold all delta data - fill in the data for + # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) # The dbuf buffer contains commands after the first two MSB sizes, the @@ -427,37 +427,37 @@ def _set_cache_brute_(self, attr): ddata.write(dbuf) # read the rest from the stream. The size we give is larger than necessary stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - + ####################################################################### if 'c_apply_delta' in globals(): c_apply_delta(bbuf, ddata, tbuf); else: apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### - - # finally, swap out source and target buffers. The target is now the + + # finally, swap out source and target buffers. The target is now the # base for the next delta to apply bbuf, tbuf = tbuf, bbuf bbuf.seek(0) tbuf.seek(0) final_target_size = target_size # END for each delta to apply - + # its already seeked to 0, constrain it to the actual size # NOTE: in the end of the loop, it swaps buffers, hence our target buffer # is not tbuf, but bbuf ! self._mm_target = bbuf self._size = final_target_size - - + + #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ else: _set_cache_ = _set_cache_too_slow_without_c - + #} END configuration - + def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: @@ -467,63 +467,63 @@ def read(self, count=0): data = self._mm_target.read(count) self._br += len(data) return data - + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading - + :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self._br = 0 self._mm_target.seek(0) - - #{ Interface - + + #{ Interface + @classmethod def new(cls, stream_list): """ Convert the given list of streams into a stream which resolves deltas when reading from it. - + :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. - - :return: Non-Delta OPackStream object whose stream can be used to obtain + + :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handled""" if len(stream_list) < 2: raise ValueError("Need at least two streams") # END single object special handling - + if stream_list[-1].type_id in delta_types: raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream - + return cls(stream_list) - + #} END interface - - + + #{ OInfo like Interface - + @property def type(self): return self._bstream.type - + @property def type_id(self): return self._bstream.type_id - + @property def size(self): """:return: number of uncompressed bytes in the stream""" return self._size - - #} END oinfo like interface - - + + #} END oinfo like interface + + #} END RO streams @@ -533,7 +533,7 @@ class Sha1Writer(object): """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" __slots__ = "sha1" - + def __init__(self): self.sha1 = make_sha() @@ -545,29 +545,29 @@ def write(self, data): self.sha1.update(data) return len(data) - # END stream interface + # END stream interface #{ Interface - + def sha(self, as_hex = False): """:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise""" if as_hex: return self.sha1.hexdigest() return self.sha1.digest() - - #} END interface + + #} END interface class FlexibleSha1Writer(Sha1Writer): - """Writer producing a sha1 while passing on the written bytes to the given + """Writer producing a sha1 while passing on the written bytes to the given write function""" __slots__ = 'writer' - + def __init__(self, writer): Sha1Writer.__init__(self) self.writer = writer - + def write(self, data): Sha1Writer.write(self, data) self.writer(data) @@ -580,18 +580,18 @@ def __init__(self): Sha1Writer.__init__(self) self.buf = StringIO() self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - + def __getattr__(self, attr): return getattr(self.buf, attr) - + def write(self, data): alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) return alen - + def close(self): self.buf.write(self.zip.flush()) - + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" @@ -599,23 +599,23 @@ def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): raise ValueError("Can only seek to position 0") # END handle offset self.buf.seek(0) - + def getvalue(self): """:return: string value from the current stream position to the end""" return self.buf.getvalue() class FDCompressedSha1Writer(Sha1Writer): - """Digests data written to it, making the sha available, then compress the + """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor - + **Note:** operates on raw file descriptors **Note:** for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") - + # default exception exc = IOError("Failed to write all bytes to filedescriptor") - + def __init__(self, fd): super(FDCompressedSha1Writer, self).__init__() self.fd = fd @@ -643,33 +643,33 @@ def close(self): class FDStream(object): - """A simple wrapper providing the most basic functions on a file descriptor + """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership""" __slots__ = ("_fd", '_pos') def __init__(self, fd): self._fd = fd self._pos = 0 - + def write(self, data): self._pos += len(data) os.write(self._fd, data) - + def read(self, count=0): if count == 0: count = os.path.getsize(self._filepath) # END handle read everything - + bytes = os.read(self._fd, count) self._pos += len(bytes) return bytes - + def fileno(self): return self._fd - + def tell(self): return self._pos - + def close(self): close(self._fd) @@ -678,17 +678,15 @@ class NullStream(object): """A stream that does nothing but providing a stream interface. Use it like /dev/null""" __slots__ = tuple() - + def read(self, size=0): return '' - + def close(self): pass - + def write(self, data): return len(data) #} END W streams - - diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index f8059447f..e84e503b6 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -5,7 +5,7 @@ import gitdb.util -#{ Initialization +#{ Initialization def _init_pool(): """Assure the pool is actually threaded""" size = 2 diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 62614ee5c..dc89039dd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -9,16 +9,16 @@ ZippedStoreShaWriter, fixture_path, TestBase - ) +) from gitdb.stream import Sha1Writer from gitdb.base import ( - IStream, - OStream, - OInfo - ) - + IStream, + OStream, + OInfo +) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type @@ -28,14 +28,14 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') - + class TestDBBase(TestBase): """Base class providing testing routines on databases""" - + # data two_lines = "1234\nhello world" all_data = (two_lines, ) - + def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() @@ -46,23 +46,23 @@ def _assert_object_writing_simple(self, db): new_istream = db.store(istream) assert new_istream is istream assert db.has_object(istream.binsha) - + info = db.info(istream.binsha) assert isinstance(info, OInfo) assert info.type == istream.type and info.size == istream.size - + stream = db.stream(istream.binsha) assert isinstance(stream, OStream) assert stream.binsha == info.binsha and stream.type == info.type assert stream.read() == data # END for each item - + assert db.size() == null_objs + ni shas = list(db.sha_iter()) assert len(shas) == db.size() assert len(shas[0]) == 20 - - + + def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW **Note:** requires write access to the database""" @@ -76,25 +76,25 @@ def _assert_object_writing(self, db): ostream = ostreamcls() assert isinstance(ostream, Sha1Writer) # END create ostream - + prev_ostream = db.set_ostream(ostream) - assert type(prev_ostream) in ostreams or prev_ostream in ostreams - + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + istream = IStream(str_blob_type, len(data), StringIO(data)) - + # store returns same istream instance, with new sha set my_istream = db.store(istream) sha = istream.binsha assert my_istream is istream assert db.has_object(sha) != dry_run - assert len(sha) == 20 - + assert len(sha) == 20 + # verify data - the slow way, we want to run code if not dry_run: info = db.info(sha) assert str_blob_type == info.type assert info.size == len(data) - + ostream = db.stream(sha) assert ostream.read() == data assert ostream.type == str_blob_type @@ -102,29 +102,29 @@ def _assert_object_writing(self, db): else: self.failUnlessRaises(BadObject, db.info, sha) self.failUnlessRaises(BadObject, db.stream, sha) - + # DIRECT STREAM COPY # our data hase been written in object format to the StringIO # we pasesd as output stream. No physical database representation # was created. - # Test direct stream copy of object streams, the result must be + # Test direct stream copy of object streams, the result must be # identical to what we fed in ostream.seek(0) istream.stream = ostream assert istream.binsha is not None prev_sha = istream.binsha - + db.set_ostream(ZippedStoreShaWriter()) db.store(istream) assert istream.binsha == prev_sha new_ostream = db.ostream() - + # note: only works as long our store write uses the same compression # level, which is zip_best assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode - + def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 @@ -134,23 +134,23 @@ def istream_generator(offset=0, ni=ni): yield IStream(str_blob_type, len(data), StringIO(data)) # END for each item # END generator utility - + # for now, we are very trusty here as we expect it to work if it worked # in the single-stream case - + # write objects reader = IteratorReader(istream_generator()) istream_reader = db.store_async(reader) istreams = istream_reader.read() # read all assert istream_reader.task().error() is None assert len(istreams) == ni - + for stream in istreams: assert stream.error is None assert len(stream.binsha) == 20 assert isinstance(stream, IStream) # END assert each stream - + # test has-object-async - we must have all previously added ones reader = IteratorReader( istream.binsha for istream in istreams ) hasobject_reader = db.has_object_async(reader) @@ -160,11 +160,11 @@ def istream_generator(offset=0, ni=ni): count += 1 # END for each sha assert count == ni - + # read the objects we have just written reader = IteratorReader( istream.binsha for istream in istreams ) ostream_reader = db.stream_async(reader) - + # read items individually to prevent hitting possible sys-limits count = 0 for ostream in ostream_reader: @@ -173,30 +173,30 @@ def istream_generator(offset=0, ni=ni): # END for each ostream assert ostream_reader.task().error() is None assert count == ni - + # get info about our items reader = IteratorReader( istream.binsha for istream in istreams ) info_reader = db.info_async(reader) - + count = 0 for oinfo in info_reader: assert isinstance(oinfo, OInfo) count += 1 # END for each oinfo instance assert count == ni - - + + # combined read-write using a converter # add 2500 items, and obtain their output streams nni = 2500 reader = IteratorReader(istream_generator(offset=ni, ni=nni)) istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - + istream_reader = db.store_async(reader) istream_reader.set_post_cb(istream_to_sha) - + ostream_reader = db.stream_async(istream_reader) - + count = 0 # read it individually, otherwise we might run into the ulimit for ostream in ostream_reader: @@ -204,5 +204,3 @@ def istream_generator(offset=0, ni=ni): count += 1 # END for each ostream assert count == nni - - diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 1ef577aa3..4894c6a79 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,15 +7,15 @@ from gitdb.db import GitDB from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex - + class TestGitDB(TestDBBase): - + def test_reading(self): gdb = GitDB(fixture_path('../../../.git/objects')) - + # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 - + # access should be possible gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) @@ -23,25 +23,25 @@ def test_reading(self): assert gdb.size() > 200 sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() - - - # This is actually a test for compound functionality, but it doesn't + + + # This is actually a test for compound functionality, but it doesn't # have a separate test module # test partial shas # this one as uneven and quite short assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") - + # mix even/uneven hexshas for i, binsha in enumerate(sha_list): assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha # END for each sha - + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") - + @with_rw_directory def test_writing(self, path): gdb = GitDB(path) - + # its possible to write objects self._assert_object_writing(gdb) self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index d7e1d01b0..e295db563 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -6,29 +6,28 @@ from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex - + class TestLooseDB(TestDBBase): - + @with_rw_directory def test_basics(self, path): ldb = LooseObjectDB(path) - + # write data self._assert_object_writing(ldb) self._assert_object_writing_async(ldb) - + # verify sha iteration and size shas = list(ldb.sha_iter()) assert shas and len(shas[0]) == 20 - + assert len(shas) == ldb.size() - + # verify find short object long_sha = bin_to_hex(shas[-1]) for short_sha in (long_sha[:20], long_sha[:5]): assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha # END for each sha - + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') # raises if no object could be foudn - diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index df428e2b7..ac9bc34e9 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -4,27 +4,27 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( - MemoryDB, - LooseObjectDB - ) - + MemoryDB, + LooseObjectDB +) + class TestMemoryDB(TestDBBase): - + @with_rw_directory def test_writing(self, path): mdb = MemoryDB() - + # write data self._assert_object_writing_simple(mdb) - + # test stream copy ldb = LooseObjectDB(path) assert ldb.size() == 0 num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) assert num_streams_copied == mdb.size() - + assert ldb.size() == mdb.size() for sha in mdb.sha_iter(): assert ldb.has_object(sha) - assert ldb.stream(sha).read() == mdb.stream(sha).read() + assert ldb.stream(sha).read() == mdb.stream(sha).read() # END verify objects where copied and are equal diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f4cb5bbc6..0d9110a01 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -12,45 +12,45 @@ import random class TestPackDB(TestDBBase): - + @with_rw_directory @with_packs_rw def test_writing(self, path): pdb = PackedDB(path) - + # on demand, we init our pack cache num_packs = len(pdb.entities()) assert pdb._st_mtime != 0 - - # test pack directory changed: + + # test pack directory changed: # packs removed - rename a file, should affect the glob pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" os.rename(pack_path, new_pack_path) - + pdb.update_cache(force=True) assert len(pdb.entities()) == num_packs - 1 - + # packs added os.rename(new_pack_path, pack_path) pdb.update_cache(force=True) assert len(pdb.entities()) == num_packs - + # bang on the cache # access the Entities directly, as there is no iteration interface # yet ( or required for now ) sha_list = list(pdb.sha_iter()) assert len(sha_list) == pdb.size() - + # hit all packs in random order random.shuffle(sha_list) - + for sha in sha_list: info = pdb.info(sha) stream = pdb.stream(sha) # END for each sha to query - - + + # test short finding - be a bit more brutal here max_bytes = 19 min_bytes = 2 @@ -64,10 +64,10 @@ def test_writing(self, path): pass # valid, we can have short objects # END exception handling # END for each sha to find - + # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... + # but in our pack, there is no ambigious ... # assert num_ambiguous - + # non-existing self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 1637bff74..086446823 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -6,14 +6,14 @@ from gitdb.db import ReferenceDB from gitdb.util import ( - NULL_BIN_SHA, - hex_to_bin - ) + NULL_BIN_SHA, + hex_to_bin +) import os - + class TestReferenceDB(TestDBBase): - + def make_alt_file(self, alt_path, alt_list): """Create an alternates file which contains the given alternates. The list can be empty""" @@ -21,40 +21,38 @@ def make_alt_file(self, alt_path, alt_list): for alt in alt_list: alt_file.write(alt + "\n") alt_file.close() - + @with_rw_directory def test_writing(self, path): NULL_BIN_SHA = '\0' * 20 - + alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) assert len(rdb.databases()) == 0 assert rdb.size() == 0 assert len(list(rdb.sha_iter())) == 0 - + # try empty, non-existing assert not rdb.has_object(NULL_BIN_SHA) - - + + # setup alternate file # add two, one is invalid own_repo_path = fixture_path('../../../.git/objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 - + # we should now find a default revision of ours gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert rdb.has_object(gitdb_sha) - + # remove valid self.make_alt_file(alt_path, ["just/one/invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 0 - + # add valid self.make_alt_file(alt_path, [own_repo_path]) rdb.update_cache() assert len(rdb.databases()) == 1 - - diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ac8473a4e..685af2fad 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,12 +4,12 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( - OStream, + OStream, ) -from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter - ) +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) from gitdb.util import zlib @@ -30,14 +30,14 @@ class TestBase(unittest.TestCase): """Base class for all tests""" - + #} END bases #{ Decorators def with_rw_directory(func): - """Create a temporary directory which can be written to, remove it if the + """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) @@ -52,7 +52,7 @@ def wrapper(self): raise finally: # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as + # a windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. if not keep: @@ -60,20 +60,20 @@ def wrapper(self): shutil.rmtree(path) # END handle exception # END wrapper - + wrapper.__name__ = func.__name__ return wrapper def with_packs_rw(func): - """Function that provides a path into which the packs for testing should be + """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" def wrapper(self, path): src_pack_glob = fixture_path('packs/*') copy_files_globbed(src_pack_glob, path, hard_link_ok=True) return func(self, path) # END wrapper - + wrapper.__name__ = func.__name__ return wrapper @@ -86,10 +86,10 @@ def fixture_path(relapath=''): :param relapath: relative path into the fixtures directory, or '' to obtain the fixture directory itself""" return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) - + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): """Copy all files found according to the given source glob into the target directory - :param hard_link_ok: if True, hard links will be created if possible. Otherwise + :param hard_link_ok: if True, hard links will be created if possible. Otherwise the files will be copied""" for src_file in glob.glob(source_glob): if hard_link_ok and hasattr(os, 'link'): @@ -103,7 +103,7 @@ def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): shutil.copy(src_file, target_dir) # END try hard link # END for each file to copy - + def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes @@ -121,7 +121,7 @@ def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata + data - + def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" @@ -137,14 +137,14 @@ def __init__(self): self.was_read = False self.bytes = 0 self.closed = False - + def read(self, size): self.was_read = True self.bytes = size - + def close(self): self.closed = True - + def _assert(self): assert self.was_read @@ -153,10 +153,9 @@ class DeriveTest(OStream): def __init__(self, sha, type, size, stream, *args, **kwargs): self.myarg = kwargs.pop('myarg') self.args = args - + def _assert(self): assert self.args assert self.myarg #} END stream utilitiess - diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index d4ce428c3..76b4d2709 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -6,7 +6,7 @@ from lib import ( TestBase, DummyStream, - DeriveTest, + DeriveTest, ) from gitdb import * @@ -20,33 +20,33 @@ class TestBaseTypes(TestBase): - + def test_streams(self): # test info sha = NULL_BIN_SHA s = 20 blob_id = 3 - + info = OInfo(sha, str_blob_type, s) assert info.binsha == sha assert info.type == str_blob_type assert info.type_id == blob_id assert info.size == s - + # test pack info # provides type_id pinfo = OPackInfo(0, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - - + + # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) @@ -56,33 +56,33 @@ def test_streams(self): assert stream.bytes == 15 ostream.read(20) assert stream.bytes == 20 - + # test packstream postream = OPackStream(*(pinfo + (stream, ))) assert postream.stream is stream postream.read(10) stream._assert() assert stream.bytes == 10 - + # test deltapackstream dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) dpostream.stream is stream dpostream.read(5) stream._assert() assert stream.bytes == 5 - + # derive with own args DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - + # test istream istream = IStream(str_blob_type, s, stream) assert istream.binsha == None istream.binsha = sha assert istream.binsha == sha - + assert len(istream.binsha) == 20 assert len(istream.hexsha) == 40 - + assert istream.size == s istream.size = s * 2 istream.size == s * 2 @@ -92,7 +92,7 @@ def test_streams(self): assert istream.stream is stream istream.stream = None assert istream.stream is None - + assert istream.error is None istream.error = Exception() assert isinstance(istream.error, Exception) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 611ae4299..f45063b1e 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -11,17 +11,17 @@ from cStringIO import StringIO from async import IteratorReader - + class TestExamples(TestBase): - + def test_base(self): ldb = LooseObjectDB(fixture_path("../../../.git/objects")) - + for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) ostream = ldb.stream(sha1) assert oinfo[:3] == ostream[:3] - + assert len(ostream.read()) == ostream.size assert ldb.has_object(oinfo.binsha) # END for each sha in database @@ -32,33 +32,33 @@ def test_base(self): except UnboundLocalError: pass # END ignore exception if there are no loose objects - + data = "my data" istream = IStream("blob", len(data), StringIO(data)) - + # the object does not yet have a sha assert istream.binsha is None ldb.store(istream) # now the sha is set assert len(istream.binsha) == 20 assert ldb.has_object(istream.binsha) - - + + # async operation # Create a reader from an iterator reader = IteratorReader(ldb.sha_iter()) - + # get reader for object streams info_reader = ldb.stream_async(reader) - + # read one info = info_reader.read(1)[0] - + # read all the rest until depletion ostreams = info_reader.read() - + # set the pool to use two threads pool.set_size(2) - + # synchronize the mode of operation pool.set_size(0) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 779155a2a..f28aef4d4 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -12,15 +12,15 @@ from gitdb.stream import DeltaApplyReader from gitdb.pack import ( - PackEntity, - PackIndexFile, - PackFile - ) + PackEntity, + PackIndexFile, + PackFile +) from gitdb.base import ( - OInfo, - OStream, - ) + OInfo, + OStream, +) from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation @@ -39,15 +39,15 @@ def bin_sha_from_filename(filename): #} END utilities class TestPack(TestBase): - + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - - + + def _assert_index_file(self, index, version, size): assert index.packfile_checksum() != index.indexfile_checksum() assert len(index.packfile_checksum()) == 20 @@ -55,93 +55,93 @@ def _assert_index_file(self, index, version, size): assert index.version() == version assert index.size() == size assert len(index.offsets()) == size - + # get all data of all objects for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) - + entry = index.entry(oidx) assert len(entry) == 3 - + assert entry[0] == index.offset(oidx) assert entry[1] == sha assert entry[2] == index.crc(oidx) - + # verify partial sha for l in (4,8,11,17,20): assert index.partial_sha_to_index(sha[:l], l*2) == oidx - + # END for each object index in indexfile self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - - + + def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 assert pack.size() == size assert len(pack.checksum()) == 20 - + num_obj = 0 for obj in pack.stream_iter(): num_obj += 1 info = pack.info(obj.pack_offset) stream = pack.stream(obj.pack_offset) - + assert info.pack_offset == stream.pack_offset assert info.type_id == stream.type_id assert hasattr(stream, 'read') - + # it should be possible to read from both streams assert obj.read() == stream.read() - + streams = pack.collect_streams(obj.pack_offset) assert streams - + # read the stream try: dstream = DeltaApplyReader.new(streams) except ValueError: - # ignore these, old git versions use only ref deltas, + # ignore these, old git versions use only ref deltas, # which we havent resolved ( as we are without an index ) # Also ignore non-delta streams continue # END get deltastream - + # read all data = dstream.read() assert len(data) == dstream.size - + # test seek dstream.seek(0) assert dstream.read() == data - - + + # read chunks # NOTE: the current implementation is safe, it basically transfers # all calls to the underlying memory map - + # END for each object assert num_obj == size - - + + def test_pack_index(self): # check version 1 and 2 - for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): index = PackIndexFile(indexfile) self._assert_index_file(index, version, size) # END run tests - + def test_pack(self): - # there is this special version 3, but apparently its like 2 ... + # there is this special version 3, but apparently its like 2 ... for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): pack = PackFile(packfile) self._assert_pack_file(pack, version, size) # END for each pack to test - + @with_rw_directory def test_pack_entity(self, rw_dir): pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo @@ -150,7 +150,7 @@ def test_pack_entity(self, rw_dir): assert entity.pack().path() == packfile assert entity.index().path() == indexfile pack_objs.extend(entity.stream_iter()) - + count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): count += 1 @@ -158,10 +158,10 @@ def test_pack_entity(self, rw_dir): assert len(info.binsha) == 20 assert info.type_id == stream.type_id assert info.size == stream.size - + # we return fully resolved items, which is implied by the sha centric access assert not info.type_id in delta_types - + # try all calls assert len(entity.collect_streams(info.binsha)) oinfo = entity.info(info.binsha) @@ -170,7 +170,7 @@ def test_pack_entity(self, rw_dir): ostream = entity.stream(info.binsha) assert isinstance(ostream, OStream) assert ostream.binsha is not None - + # verify the stream try: assert entity.is_valid_stream(info.binsha, use_crc=True) @@ -180,16 +180,16 @@ def test_pack_entity(self, rw_dir): assert entity.is_valid_stream(info.binsha, use_crc=False) # END for each info, stream tuple assert count == size - + # END for each entity - + # pack writing - write all packs into one # index path can be None pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 def rewind_streams(): - for obj in pack_objs: + for obj in pack_objs: obj.stream.seek(0) #END utility for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): @@ -199,23 +199,23 @@ def rewind_streams(): ifile = open(ipath, 'wb') iwrite = ifile.write #END handle ip - + # make sure we rewind the streams ... we work on the same objects over and over again - if iteration > 0: + if iteration > 0: rewind_streams() #END rewind streams iteration += 1 - + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) pfile.close() assert os.path.getsize(ppath) > 100 - + # verify pack pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha - + # verify index if ipath is not None: ifile.close() @@ -227,7 +227,7 @@ def rewind_streams(): assert idx.size() == len(pack_objs) #END verify files exist #END for each packpath, indexpath pair - + # verify the packs throughly rewind_streams() entity = PackEntity.create(pack_objs, rw_dir) @@ -239,9 +239,9 @@ def rewind_streams(): # END for each crc mode #END for each info assert count == len(pack_objs) - - + + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets - # of course without really needing such a huge pack + # of course without really needing such a huge pack raise SkipTest() diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 6dc27463c..8360ea36c 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -32,18 +32,18 @@ class TestStream(TestBase): """Test stream classes""" - + data_sizes = (15, 10000, 1000*1024+512) - + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): - """Make stream tests - the orig_stream is seekable, allowing it to be + """Make stream tests - the orig_stream is seekable, allowing it to be rewound and reused :param cdata: the data we expect to read from stream, the contents :param rewind_stream: function called to rewind the stream to make it ready for reuse""" ns = 10 assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) - + # read in small steps ss = len(cdata) / ns for i in range(ns): @@ -55,30 +55,30 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): if rest: assert rest == cdata[-len(rest):] # END handle rest - + if isinstance(stream, DecompressMemMapReader): assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type - + rewind_stream(stream) - + # read everything rdata = stream.read() assert rdata == cdata - + if isinstance(stream, DecompressMemMapReader): assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type - + def test_decompress_reader(self): for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: cdata = make_bytes(ds, randomize=False) - + # zdata = zipped actual data # cdata = original content data - + # create reader if with_size: # need object data @@ -86,7 +86,7 @@ def test_decompress_reader(self): type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) assert type == str_blob_type - + # even if we don't set the size, it will be set automatically on first read test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) assert test_reader._s == len(cdata) @@ -95,60 +95,59 @@ def test_decompress_reader(self): zdata = zlib.compress(cdata) reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) assert reader._s == len(cdata) - # END get reader - + # END get reader + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) - + # put in a dummy stream for closing dummy = DummyStream() reader._m = dummy - + assert not dummy.closed del(reader) assert dummy.closed == close_on_deletion # END for each datasize # END whether size should be used # END whether stream should be closed when deleted - + def test_sha_writer(self): writer = Sha1Writer() assert 2 == writer.write("hi") assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 - + # make sure it does something ;) prev_sha = writer.sha() writer.write("hi again") assert writer.sha() != prev_sha - + def test_compressed_writer(self): for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) data = make_bytes(ds, randomize=False) - + # for now, just a single write, code doesn't care about chunking assert len(data) == ostream.write(data) ostream.close() - + # its closed already self.failUnlessRaises(OSError, os.close, fd) - + # read everything back, compare to data we zip fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) assert len(written_data) == os.path.getsize(path) os.close(fd) assert written_data == zlib.compress(data, 1) # best speed - + os.remove(path) # END for each os - + def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) - + # if there is a bug, we will be missing one byte exactly ! data = ostream.read() assert len(data) == ostream.size - diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 35f9f44a7..ed69f0d1f 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -8,28 +8,28 @@ from lib import TestBase from gitdb.util import ( - to_hex_sha, - to_bin_sha, - NULL_HEX_SHA, + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, LockedFD - ) +) + - class TestUtils(TestBase): def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA - + def _cmp_contents(self, file_path, data): - # raise if data from file at file_path + # raise if data from file at file_path # does not match data string fp = open(file_path, "rb") try: assert fp.read() == data finally: fp.close() - + def test_lockedfd(self): my_file = tempfile.mktemp() orig_data = "hello" @@ -37,43 +37,43 @@ def test_lockedfd(self): my_file_fp = open(my_file, "wb") my_file_fp.write(orig_data) my_file_fp.close() - + try: lfd = LockedFD(my_file) - lockfilepath = lfd._lockfilepath() - + lockfilepath = lfd._lockfilepath() + # cannot end before it was started self.failUnlessRaises(AssertionError, lfd.rollback) self.failUnlessRaises(AssertionError, lfd.commit) - + # open for writing assert not os.path.isfile(lockfilepath) wfd = lfd.open(write=True) assert lfd._fd is wfd assert os.path.isfile(lockfilepath) - + # write data and fail os.write(wfd, new_data) lfd.rollback() assert lfd._fd is None self._cmp_contents(my_file, orig_data) assert not os.path.isfile(lockfilepath) - + # additional call doesnt fail lfd.commit() lfd.rollback() - + # test reading lfd = LockedFD(my_file) rfd = lfd.open(write=False) assert os.read(rfd, len(orig_data)) == orig_data - + assert os.path.isfile(lockfilepath) # deletion rolls back del(lfd) assert not os.path.isfile(lockfilepath) - - + + # write data - concurrently lfd = LockedFD(my_file) olfd = LockedFD(my_file) @@ -82,17 +82,17 @@ def test_lockedfd(self): assert os.path.isfile(lockfilepath) # another one fails self.failUnlessRaises(IOError, olfd.open) - + wfdstream.write(new_data) lfd.commit() assert not os.path.isfile(lockfilepath) self._cmp_contents(my_file, new_data) - + # could test automatic _end_writing on destruction finally: os.remove(my_file) # END final cleanup - + # try non-existing file for reading lfd = LockedFD(tempfile.mktemp()) try: @@ -102,4 +102,3 @@ def test_lockedfd(self): else: self.fail("expected OSError") # END handle exceptions - diff --git a/gitdb/util.py b/gitdb/util.py index 1662b662d..b167b4d12 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -24,10 +24,9 @@ from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, SlidingWindowMapManager, + SlidingWindowMapBuffer +) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -60,7 +59,7 @@ def unpack_from(fmt, data, offset=0): #{ Globals -# A pool distributing tasks, initially with zero threads, hence everything +# A pool distributing tasks, initially with zero threads, hence everything # will be handled in the main thread pool = ThreadPool(0) @@ -97,35 +96,35 @@ def unpack_from(fmt, data, offset=0): #} END Aliases -#{ compatibility stuff ... +#{ compatibility stuff ... class _RandomAccessStringIO(object): - """Wrapper to provide required functionality in case memory maps cannot or may + """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' - + def __init__(self, buf=''): self._sio = StringIO(buf) - + def __getattr__(self, attr): return getattr(self._sio, attr) - + def __len__(self): return len(self.getvalue()) - + def __getitem__(self, i): return self.getvalue()[i] - + def __getslice__(self, start, end): return self.getvalue()[start:end] - + #} END compatibility stuff ... #{ Routines def make_sha(source=''): """A python2.4 workaround for the sha/hashlib module fiasco - + **Note** From the dulwich project """ try: return hashlib.sha1(source) @@ -138,25 +137,25 @@ def allocate_memory(size): if size == 0: return _RandomAccessStringIO('') # END handle empty chunks gracefully - + try: return mmap.mmap(-1, size) # read-write by default except EnvironmentError: # setup real memory instead # this of course may fail if the amount of memory is not available in - # one chunk - would only be the case in python 2.4, being more likely on + # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. return _RandomAccessStringIO("\0"*size) # END handle memory allocation - + def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd - + :param fd: file descriptor opened for reading :param stream: if False, random access is provided, otherwise the stream interface is provided. - :param allow_mmap: if True, its allowed to map the contents into memory, which + :param allow_mmap: if True, its allowed to map the contents into memory, which allows large files to be handled and accessed efficiently. The file-descriptor will change its position if this is False""" try: @@ -171,24 +170,24 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): except OSError: pass # END exception handling - + # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: return _RandomAccessStringIO(contents) return contents - + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible - + :return: random access compatible memory of the given filepath :param stream: see ``file_contents_ro`` :param allow_mmap: see ``file_contents_ro`` :param flags: additional flags to pass to os.open :raise OSError: If the file could not be opened - - **Note** for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: @@ -196,19 +195,19 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): finally: close(fd) # END assure file is closed - + def sliding_ro_buffer(filepath, flags=0): """ :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) - + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: return sha return bin_to_hex(sha) - + def to_bin_sha(sha): if len(sha) == 20: return sha @@ -227,12 +226,12 @@ class LazyMixin(object): is actually accessed and retrieved the first time. All future accesses will return the cached value as stored in the Instance's dict or slot. """ - + __slots__ = tuple() - + def __getattr__(self, attr): """ - Whenever an attribute is requested that we do not know, we allow it + Whenever an attribute is requested that we do not know, we allow it to be created and set. Next time the same attribute is reqeusted, it is simply returned from our dict/slots. """ self._set_cache_(attr) @@ -241,65 +240,65 @@ def __getattr__(self, attr): def _set_cache_(self, attr): """ - This method should be overridden in the derived class. + This method should be overridden in the derived class. It should check whether the attribute named by attr can be created and cached. Do nothing if you do not know the attribute or call your subclass - - The derived class may create as many additional attributes as it deems - necessary in case a git command returns more information than represented + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented in the single attribute.""" pass - + class LockedFD(object): """ This class facilitates a safe read and write operation to a file on disk. - If we write to 'file', we obtain a lock file at 'file.lock' and write to - that instead. If we succeed, the lock file will be renamed to overwrite + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite the original file. - - When reading, we obtain a lock file, but to prevent other writers from + + When reading, we obtain a lock file, but to prevent other writers from succeeding while we are reading the file. - - This type handles error correctly in that it will assure a consistent state + + This type handles error correctly in that it will assure a consistent state on destruction. - + **note** with this setup, parallel reading is not possible""" __slots__ = ("_filepath", '_fd', '_write') - + def __init__(self, filepath): """Initialize an instance with the givne filepath""" self._filepath = filepath self._fd = None self._write = None # if True, we write a file - + def __del__(self): # will do nothing if the file descriptor is already closed if self._fd is not None: self.rollback() - + def _lockfilepath(self): return "%s.lock" % self._filepath - + def open(self, write=False, stream=False): """ Open the file descriptor for reading or writing, both in binary mode. - + :param write: if True, the file descriptor will be opened for writing. Other wise it will be opened read-only. - :param stream: if True, the file descriptor will be wrapped into a simple stream + :param stream: if True, the file descriptor will be wrapped into a simple stream object which supports only reading or writing :return: fd to read from or write to. It is still maintained by this instance and must not be closed directly :raise IOError: if the lock could not be retrieved :raise OSError: If the actual file could not be opened for reading - + **note** must only be called once""" if self._write is not None: raise AssertionError("Called %s multiple times" % self.open) - + self._write = write - + # try to open the lock file binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary @@ -313,7 +312,7 @@ def open(self, write=False, stream=False): except OSError: raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) # END handle lock retrieval - + # open actual file if required if self._fd is None: # we could specify exlusive here, as we obtained the lock anyway @@ -325,7 +324,7 @@ def open(self, write=False, stream=False): raise # END handle lockfile # END open descriptor for reading - + if stream: # need delayed import from stream import FDStream @@ -333,33 +332,33 @@ def open(self, write=False, stream=False): else: return self._fd # END handle stream - + def commit(self): - """When done writing, call this function to commit your changes into the - actual file. + """When done writing, call this function to commit your changes into the + actual file. The file descriptor will be closed, and the lockfile handled. - + **Note** can be called multiple times""" self._end_writing(successful=True) - + def rollback(self): - """Abort your operation without any changes. The file descriptor will be + """Abort your operation without any changes. The file descriptor will be closed, and the lock released. - + **Note** can be called multiple times""" self._end_writing(successful=False) - + def _end_writing(self, successful=True): """Handle the lock according to the write mode """ if self._write is None: raise AssertionError("Cannot end operation if it wasn't started yet") - + if self._fd is None: return - + os.close(self._fd) self._fd = None - + lockfile = self._lockfilepath() if self._write and successful: # on windows, rename does not silently overwrite the existing one @@ -369,7 +368,7 @@ def _end_writing(self, successful=True): # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) - + # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well From b6c493deb2341fb843d71b66b2aa23078638755c Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:41:15 -0400 Subject: [PATCH 0235/1790] Pick off the low hanging fruit This fixes most of the import errors that came from using the implicit relative imports that Python 2 supports. This also fixes the use of `xrange`, which has replaced `range` in Python 3. The same has happened for `izip`, which is also being aliased. The octal number syntax changed in Python 3, so we are now converting from strings using the `int` built-in function, which will produce the same output across both versions of Python. --- gitdb/__init__.py | 7 ++- gitdb/base.py | 18 +++---- gitdb/db/__init__.py | 13 +++-- gitdb/db/base.py | 1 + gitdb/db/git.py | 18 +++---- gitdb/db/loose.py | 20 ++++---- gitdb/db/mem.py | 23 ++++++--- gitdb/db/pack.py | 14 +++--- gitdb/db/ref.py | 10 ++-- gitdb/exc.py | 2 +- gitdb/fun.py | 21 +++++++-- gitdb/pack.py | 94 ++++++++++++++++++++----------------- gitdb/stream.py | 43 ++++++++++------- gitdb/test/__init__.py | 2 +- gitdb/test/db/lib.py | 12 ++++- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_mem.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 11 ++++- gitdb/test/test_base.py | 2 +- gitdb/test/test_example.py | 12 +++-- gitdb/test/test_pack.py | 23 +++++---- gitdb/test/test_stream.py | 16 +++---- gitdb/test/test_util.py | 2 +- gitdb/util.py | 18 +++---- 27 files changed, 228 insertions(+), 164 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 847269a33..66d1b1c40 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -32,7 +32,6 @@ def _init_externals(): # default imports -from db import * -from base import * -from stream import * - +from gitdb.base import * +from gitdb.db import * +from gitdb.stream import * diff --git a/gitdb/base.py b/gitdb/base.py index a673c2376..1eb423284 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -3,15 +3,15 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" -from util import ( - bin_to_hex, - zlib - ) - -from fun import ( - type_id_to_type_map, - type_to_type_id_map - ) +from gitdb.util import ( + bin_to_hex, + zlib +) + +from gitdb.fun import ( + type_id_to_type_map, + type_to_type_id_map +) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index e5935b7c2..0a2a46a64 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -3,10 +3,9 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import * -from loose import * -from mem import * -from pack import * -from git import * -from ref import * - +from gitdb.db.base import * +from gitdb.db.loose import * +from gitdb.db.mem import * +from gitdb.db.pack import * +from gitdb.db.git import * +from gitdb.db.ref import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 0eef1e5d5..85df324f7 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -20,6 +20,7 @@ ) from itertools import chain +from functools import reduce __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 6e6ec5d1f..5c74a2049 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -2,15 +2,15 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - CompoundDB, - ObjectDBW, - FileDBBase - ) - -from loose import LooseObjectDB -from pack import PackedDB -from ref import ReferenceDB +from gitdb.db.base import ( + CompoundDB, + ObjectDBW, + FileDBBase +) + +from gitdb.db.loose import LooseObjectDB +from gitdb.db.pack import PackedDB +from gitdb.db.ref import ReferenceDB from gitdb.util import LazyMixin from gitdb.exc import ( diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4ebca84d3..ac1b9d1d6 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -2,11 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - FileDBBase, - ObjectDBR, - ObjectDBW - ) +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + ObjectDBW +) from gitdb.exc import ( @@ -69,11 +69,11 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): # On windows we need to keep it writable, otherwise it cannot be removed # either - new_objects_mode = 0444 + new_objects_mode = int("444", 8) if os.name == 'nt': - new_objects_mode = 0644 - - + new_objects_mode = int("644", 8) + + def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) self._hexsha_to_file = dict() @@ -133,7 +133,7 @@ def _map_loose_object(self, sha): db_path = self.db_path(self.object_path(bin_to_hex(sha))) try: return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) - except OSError,e: + except OSError as e: if e.errno != ENOENT: # try again without noatime try: diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index e4fba94b3..3847c34df 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -3,11 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains the MemoryDatabase implementation""" -from loose import LooseObjectDB -from base import ( - ObjectDBR, - ObjectDBW - ) +from gitdb.db.loose import LooseObjectDB +from gitdb.db.base import ( + ObjectDBR, + ObjectDBW +) from gitdb.base import ( OStream, @@ -19,7 +19,18 @@ UnsupportedOperation ) -from cStringIO import StringIO +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, +) + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO __all__ = ("MemoryDB", ) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 09f811847..eca02bbff 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -3,11 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" -from base import ( - FileDBBase, - ObjectDBR, - CachingDB - ) +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + CachingDB +) from gitdb.util import LazyMixin @@ -19,6 +19,8 @@ from gitdb.pack import PackEntity +from functools import reduce + import os import glob @@ -104,7 +106,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in xrange(index.size()): + for index in range(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 368ab9a61..748f7c145 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,9 +2,9 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - CompoundDB, - ) +from gitdb.db.base import ( + CompoundDB, +) import os __all__ = ('ReferenceDB', ) @@ -33,7 +33,7 @@ def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls if dbcls is None: # late import - from git import GitDB + from gitdb.db.git import GitDB dbcls = GitDB # END get db type @@ -68,7 +68,7 @@ def _update_dbs_from_ref_file(self): db.databases() # END verification self._dbs.append(db) - except Exception, e: + except Exception: # ignore invalid paths or issues pass # END for each path to add diff --git a/gitdb/exc.py b/gitdb/exc.py index 47fc80912..73f84d299 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with common exceptions""" -from util import to_hex_sha +from gitdb.util import to_hex_sha class ODBError(Exception): """All errors thrown by the object database""" diff --git a/gitdb/fun.py b/gitdb/fun.py index 9e5c44a6c..ce55438d1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -6,17 +6,28 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from exc import ( +from gitdb.exc import ( BadObjectType - ) +) -from util import zlib +from gitdb.util import zlib decompressobj = zlib.decompressobj import mmap -from itertools import islice, izip +from itertools import islice + +try: + from itertools import izip +except ImportError: + izip = zip -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO # INVARIANTS OFS_DELTA = 6 diff --git a/gitdb/pack.py b/gitdb/pack.py index 4a0badccb..aea0d1e5a 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -4,31 +4,32 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) -from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + BadObject, + UnsupportedOperation, + ParseError +) -from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) +from gitdb.util import ( + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, +) + +from gitdb.fun import ( + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size +) try: from _perf import PackIndexFile_sha_to_index @@ -36,22 +37,23 @@ pass # END try c module -from base import ( # Amazing ! - OInfo, - OStream, - OPackInfo, - OPackStream, - ODeltaStream, - ODeltaPackInfo, - ODeltaPackStream, - ) -from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) +from gitdb.base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, +) + +from gitdb.stream import ( + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer +) from struct import ( pack, @@ -60,7 +62,11 @@ from binascii import crc32 -from itertools import izip +try: + from itertools import izip +except ImportError: + izip = zip + import tempfile import array import os @@ -200,7 +206,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in xrange(255): + for i in range(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i+1] += v @@ -407,7 +413,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in xrange(self.size())) + return tuple(self.offset(index) for index in range(self.size())) # END handle version def sha_to_index(self, sha): @@ -694,7 +700,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in xrange(self._index.size()): + for index in range(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/stream.py b/gitdb/stream.py index b21c39c9d..52b54af50 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,28 +3,35 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + import errno import mmap import os -from fun import ( - msb_size, - stream_copy, - apply_delta_data, - connect_deltas, - DeltaChunkList, - delta_types - ) - -from util import ( - allocate_memory, - LazyMixin, - make_sha, - write, - close, - zlib - ) +from gitdb.fun import ( + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + DeltaChunkList, + delta_types +) + +from gitdb.util import ( + allocate_memory, + LazyMixin, + make_sha, + write, + close, + zlib +) has_perf_mod = False try: diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index e84e503b6..ca104c0c5 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -9,7 +9,7 @@ def _init_pool(): """Assure the pool is actually threaded""" size = 2 - print "Setting ThreadPool to %i" % size + print("Setting ThreadPool to %i" % size) gitdb.util.pool.set_size(size) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index dc89039dd..18b22ff21 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,15 @@ from gitdb.typ import str_blob_type from async import IteratorReader -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + from struct import pack @@ -40,7 +48,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in xrange(ni): + for i in range(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), StringIO(data)) new_istream = db.store(istream) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 4894c6a79..cce2b9c09 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index e295db563..5e42b639a 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index ac9bc34e9..9235b21d3 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import ( MemoryDB, LooseObjectDB diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 0d9110a01..f5a4dcb92 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import PackedDB from gitdb.test.lib import fixture_path diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 086446823..752c31de5 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import ReferenceDB from gitdb.util import ( diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 685af2fad..3ac7142c1 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -16,7 +16,14 @@ import sys import random from array import array -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO import glob import unittest @@ -109,7 +116,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes / 4 - producer = xrange(actual_size) + producer = range(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 76b4d2709..4cca7dabe 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" -from lib import ( +from gitdb.test.lib import ( TestBase, DummyStream, DeriveTest, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index f45063b1e..f57cc5029 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,12 +3,18 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from lib import * +from gitdb.test.lib import * from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool - -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO from async import IteratorReader diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index f28aef4d4..bcda3cfb8 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -3,12 +3,13 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" -from lib import ( - TestBase, - with_rw_directory, - with_packs_rw, - fixture_path - ) +from gitdb.test.lib import ( + TestBase, + with_rw_directory, + with_packs_rw, + fixture_path +) + from gitdb.stream import DeltaApplyReader from gitdb.pack import ( @@ -25,7 +26,13 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from itertools import izip, chain +from itertools import chain + +try: + from itertools import izip +except ImportError: + izip = zip + from nose import SkipTest import os @@ -57,7 +64,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in xrange(index.size()): + for oidx in range(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 8360ea36c..53aa8e21d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -3,14 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" -from lib import ( - TestBase, - DummyStream, - Sha1Writer, - make_bytes, - make_object, - fixture_path - ) +from gitdb.test.lib import ( + TestBase, + DummyStream, + Sha1Writer, + make_bytes, + make_object, + fixture_path +) from gitdb import * from gitdb.util import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index ed69f0d1f..4672dd604 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -6,7 +6,7 @@ import tempfile import os -from lib import TestBase +from gitdb.test.lib import TestBase from gitdb.util import ( to_hex_sha, to_bin_sha, diff --git a/gitdb/util.py b/gitdb/util.py index b167b4d12..3dcf3d7dc 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,13 +8,13 @@ import sys import errno -from cStringIO import StringIO - -# in py 2.4, StringIO is only StringI, without write support. -# Hence we must use the python implementation for this -if sys.version_info[1] < 5: - from StringIO import StringIO -# END handle python 2.4 +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO try: import async.mod.zlib as zlib @@ -303,7 +303,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode, 0600) + fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) if not write: os.close(fd) else: @@ -372,7 +372,7 @@ def _end_writing(self, successful=True): # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well - chmod(self._filepath, 0644) + chmod(self._filepath, int("644", 8)) else: # just delete the file so far, we failed os.remove(lockfile) From 5bfa7e6cb0872c815720f7e861393125ad0855e7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:45:51 -0400 Subject: [PATCH 0236/1790] Temporarily switch out async for testing This will be switched back when the pull request for Python 3 support has been merged into the central async repository. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 978105388..c28387570 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = https://github.com/gitpython-developers/async.git + url = https://github.com/kevin-brown/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git From 85f2b9baf1c6a5738967f8e77d2a65a1a842bde3 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:48:03 -0400 Subject: [PATCH 0237/1790] Test against Python 3.4 If it works in Python 3.3, it should also work in Python 3.4. Considering it is the latest stable release, gitdb should be tested against it. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ff263f20d..cf1d13666 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.6" - "2.7" - "3.3" + - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) script: nosetests From b881134ec816d2a54f6e8deced8db25b4bd5baa7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 16:10:54 -0400 Subject: [PATCH 0238/1790] Convert strings to bytes for PY3 In Python 3, the default string type is now the Python 2 unicode strings. The unicode strings cannot be converted to a byte stream, so we have to convert it before writing to the streams. --- gitdb/test/lib.py | 6 +++--- gitdb/test/test_stream.py | 9 ++++----- gitdb/test/test_util.py | 13 +++++++------ gitdb/util.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 3ac7142c1..f52cf79d4 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -54,7 +54,7 @@ def wrapper(self): try: return func(self, path) except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + sys.stderr.write("Test %s.%s failed, output is at %r\n" % (type(self).__name__, func.__name__, path)) keep = True raise finally: @@ -115,7 +115,7 @@ def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" - actual_size = size_in_bytes / 4 + actual_size = size_in_bytes // 4 producer = range(actual_size) if randomize: producer = list(producer) @@ -127,7 +127,7 @@ def make_bytes(size_in_bytes, randomize=False): def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) - return odata + data + return odata.encode("ascii") + data def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 53aa8e21d..f6eb371bd 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -3,6 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" + from gitdb.test.lib import ( TestBase, DummyStream, @@ -16,20 +17,18 @@ from gitdb.util import ( NULL_HEX_SHA, hex_to_bin - ) +) from gitdb.util import zlib from gitdb.typ import ( str_blob_type - ) +) import time import tempfile import os - - class TestStream(TestBase): """Test stream classes""" @@ -45,7 +44,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) # read in small steps - ss = len(cdata) / ns + ss = len(cdata) // ns for i in range(ns): data = stream.read(ss) chunk = cdata[i*ss:(i+1)*ss] diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 4672dd604..ec9a86ce7 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -5,6 +5,7 @@ """Test for object db""" import tempfile import os +import sys from gitdb.test.lib import TestBase from gitdb.util import ( @@ -19,14 +20,14 @@ class TestUtils(TestBase): def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA.encode("ascii") def _cmp_contents(self, file_path, data): # raise if data from file at file_path # does not match data string fp = open(file_path, "rb") try: - assert fp.read() == data + assert fp.read() == data.encode("ascii") finally: fp.close() @@ -35,7 +36,7 @@ def test_lockedfd(self): orig_data = "hello" new_data = "world" my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data) + my_file_fp.write(orig_data.encode("ascii")) my_file_fp.close() try: @@ -53,7 +54,7 @@ def test_lockedfd(self): assert os.path.isfile(lockfilepath) # write data and fail - os.write(wfd, new_data) + os.write(wfd, new_data.encode("ascii")) lfd.rollback() assert lfd._fd is None self._cmp_contents(my_file, orig_data) @@ -66,7 +67,7 @@ def test_lockedfd(self): # test reading lfd = LockedFD(my_file) rfd = lfd.open(write=False) - assert os.read(rfd, len(orig_data)) == orig_data + assert os.read(rfd, len(orig_data)) == orig_data.encode("ascii") assert os.path.isfile(lockfilepath) # deletion rolls back @@ -83,7 +84,7 @@ def test_lockedfd(self): # another one fails self.failUnlessRaises(IOError, olfd.open) - wfdstream.write(new_data) + wfdstream.write(new_data.encode("ascii")) lfd.commit() assert not os.path.isfile(lockfilepath) self._cmp_contents(my_file, new_data) diff --git a/gitdb/util.py b/gitdb/util.py index 3dcf3d7dc..5ea26d5bd 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -327,7 +327,7 @@ def open(self, write=False, stream=False): if stream: # need delayed import - from stream import FDStream + from gitdb.stream import FDStream return FDStream(self._fd) else: return self._fd From 01e40b5e02e90ccac06e3b0ec0adf1f8f4e48ebd Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 17:29:58 -0400 Subject: [PATCH 0239/1790] Use memoryview instead of buffer This uses memoryview by default, which is supported in Python 3 and Python 2.7, but not Python 2.6, and falls back to the old `buffer` type in Python 2.6 and when the memoryview does not support the type, such as when mmap instaces are passed in. --- gitdb/stream.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 52b54af50..a099eeba0 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -270,7 +270,10 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) + try: + indata = memoryview(self._m)[self._cws:self._cwe].tobytes() + except (NameError, TypeError): + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) From 0269405121d7ef065f7008c9c033e95e734f029a Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 18:20:48 -0400 Subject: [PATCH 0240/1790] Better handling of bytes This adds a `byte_ord` version of `ord` which will let `bytes` safely pass through in Python 3. `cmp` was also swapped out as it has been dropped in Python 3. --- gitdb/fun.py | 10 +++++----- gitdb/pack.py | 26 ++++++++++++++------------ gitdb/util.py | 15 ++++++++++++++- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index ce55438d1..8f2d46342 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -10,7 +10,7 @@ BadObjectType ) -from gitdb.util import zlib +from gitdb.util import byte_ord, zlib decompressobj = zlib.decompressobj import mmap @@ -411,13 +411,13 @@ def pack_object_header_info(data): The type_id should be interpreted according to the ``type_id_to_type_map`` map The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" - c = ord(data[0]) # first byte + c = byte_ord(data[0]) # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size while c & 0x80: - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 @@ -668,12 +668,12 @@ def is_equal_canonical_sha(canonical_length, match, sha1): hence the comparison will only use the last 4 bytes for uneven canonical representations :param match: less than 20 byte sha :param sha1: 20 byte sha""" - binary_length = canonical_length/2 + binary_length = canonical_length // 2 if match[:binary_length] != sha1[:binary_length]: return False if canonical_length - binary_length and \ - (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match)-1])) & 0xf0: return False # END handle uneven canonnical length return True diff --git a/gitdb/pack.py b/gitdb/pack.py index aea0d1e5a..4a9ee9ffa 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -15,6 +15,7 @@ LazyMixin, unpack_from, bin_to_hex, + byte_ord, ) from gitdb.fun import ( @@ -421,7 +422,7 @@ def sha_to_index(self, sha): :return: index usable with the ``offset`` or ``entry`` method, or None if the sha was not found in this pack index :param sha: 20 byte sha to lookup""" - first_byte = ord(sha[0]) + first_byte = byte_ord(sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: @@ -430,11 +431,11 @@ def sha_to_index(self, sha): # bisect until we have the sha while lo < hi: - mid = (lo + hi) / 2 - c = cmp(sha, get_sha(mid)) - if c < 0: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if sha < mid_sha: hi = mid - elif not c: + elif sha == mid_sha: return mid else: lo = mid + 1 @@ -453,7 +454,8 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - first_byte = ord(partial_bin_sha[0]) + first_byte = byte_ord(partial_bin_sha[0]) + get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: @@ -461,15 +463,15 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) + filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha)) # find lowest while lo < hi: - mid = (lo + hi) / 2 - c = cmp(filled_sha, get_sha(mid)) - if c < 0: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if filled_sha < mid_sha: hi = mid - elif not c: + elif filled_sha == mid_sha: # perfect match lo = mid break @@ -482,7 +484,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size(): + if lo + 1 < self.size(): next_sha = get_sha(lo+1) if next_sha and next_sha == cur_sha: raise AmbiguousObjectName(partial_bin_sha) diff --git a/gitdb/util.py b/gitdb/util.py index 5ea26d5bd..ed2dc3723 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -118,14 +118,27 @@ def __getitem__(self, i): def __getslice__(self, start, end): return self.getvalue()[start:end] +def byte_ord(b): + """ + Return the integer representation of the byte string. This supports Python + 3 byte arrays as well as standard strings. + """ + try: + return ord(b) + except TypeError: + return b + #} END compatibility stuff ... #{ Routines -def make_sha(source=''): +def make_sha(source=None): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ + if source is None: + source = "".encode("ascii") + try: return hashlib.sha1(source) except NameError: From d8405ee00bfc1ebdae7c41b45f8c374902a3d2b4 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 19:55:00 -0400 Subject: [PATCH 0241/1790] More bytes handling --- gitdb/db/base.py | 10 ++++++++++ gitdb/fun.py | 2 +- gitdb/pack.py | 6 +++++- gitdb/stream.py | 4 ++-- gitdb/test/db/test_ref.py | 2 +- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 85df324f7..aa7a7ee30 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,6 +22,8 @@ from itertools import chain from functools import reduce +import sys + __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -176,6 +178,14 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" + if sys.version_info[0] == 3: + text_type = str + else: + text_type = basestring + + if not isinstance(rela_path, text_type): + rela_path = rela_path.decode("utf-8") + return join(self._root_path, rela_path) #} END interface diff --git a/gitdb/fun.py b/gitdb/fun.py index 8f2d46342..9baeac3ec 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -402,7 +402,7 @@ def loose_object_header_info(m): :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0")].split(" ") + type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) return type_name, int(size) def pack_object_header_info(data): diff --git a/gitdb/pack.py b/gitdb/pack.py index 4a9ee9ffa..ad5eccb8b 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -121,7 +121,11 @@ def pack_object_at(cursor, offset, as_stream): abs_data_offset = offset + total_rela_offset if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + try: + buff = memoryview(data)[total_rela_offset:].tobytes() + except (NameError, TypeError): + buff = buffer(data, total_rela_offset) + stream = DecompressMemMapReader(buff, False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: diff --git a/gitdb/stream.py b/gitdb/stream.py index a099eeba0..75993db51 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -105,8 +105,8 @@ def _parse_header_info(self): maxb = 512 # should really be enough, cgit uses 8192 I believe self._s = maxb hdr = self.read(maxb) - hdrend = hdr.find("\0") - type, size = hdr[:hdrend].split(" ") + hdrend = hdr.find("\0".encode("ascii")) + type, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 752c31de5..e303fe2f8 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -19,7 +19,7 @@ def make_alt_file(self, alt_path, alt_list): The list can be empty""" alt_file = open(alt_path, "wb") for alt in alt_list: - alt_file.write(alt + "\n") + alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) alt_file.close() @with_rw_directory From f0b3e7bcc5278208294f40aa580ccb378ed1c165 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 20:14:24 -0400 Subject: [PATCH 0242/1790] Bytes for everyone! --- gitdb/db/loose.py | 8 ++++---- gitdb/stream.py | 2 +- gitdb/test/db/test_ref.py | 3 +-- gitdb/test/test_stream.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index ac1b9d1d6..656841651 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -16,10 +16,10 @@ ) from gitdb.stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - FDStream, - Sha1Writer + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer ) from gitdb.base import ( diff --git a/gitdb/stream.py b/gitdb/stream.py index 75993db51..1bd7fe2b2 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -551,7 +551,7 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" + :return: length of incoming data""" self.sha1.update(data) return len(data) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index e303fe2f8..a1387ee26 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -24,7 +24,7 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - NULL_BIN_SHA = '\0' * 20 + NULL_BIN_SHA = '\0'.encode("ascii") * 20 alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) @@ -35,7 +35,6 @@ def test_writing(self, path): # try empty, non-existing assert not rdb.has_object(NULL_BIN_SHA) - # setup alternate file # add two, one is invalid own_repo_path = fixture_path('../../../.git/objects') # use own repo diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f6eb371bd..f409f1788 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -111,13 +111,13 @@ def test_decompress_reader(self): def test_sha_writer(self): writer = Sha1Writer() - assert 2 == writer.write("hi") + assert 2 == writer.write("hi".encode("ascii")) assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 # make sure it does something ;) prev_sha = writer.sha() - writer.write("hi again") + writer.write("hi again".encode("ascii")) assert writer.sha() != prev_sha def test_compressed_writer(self): From a19a169ffd81e21f472dee8a9a38ac1c9fab9bd7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 21:04:47 -0400 Subject: [PATCH 0243/1790] Can't compare memoryview instances, convert to bytes --- gitdb/pack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index ad5eccb8b..d8f32c1c2 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -118,7 +118,6 @@ def pack_object_at(cursor, offset, as_stream): # assume its a base object total_rela_offset = data_rela_offset # END handle type id - abs_data_offset = offset + total_rela_offset if as_stream: try: @@ -129,6 +128,8 @@ def pack_object_at(cursor, offset, as_stream): if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: + if hasattr(delta_info, "tobytes"): + delta_info = delta_info.tobytes() return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: From 087803ee30456c4942d9c18d82c1d686eb081a27 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 21:49:51 -0400 Subject: [PATCH 0244/1790] Making a bit of progress... This changes the internals to use BytesIO over StringIO, which fixed a few of the failing tests in Python 3. We are only importing from `io` now, instead of the entire chain, as this is available in Python 2.6+. --- gitdb/db/mem.py | 8 +------- gitdb/fun.py | 10 ++-------- gitdb/stream.py | 26 ++++++++++---------------- gitdb/test/db/lib.py | 10 ++-------- gitdb/test/lib.py | 8 +------- gitdb/test/test_example.py | 8 +------- gitdb/test/test_stream.py | 4 ++-- gitdb/util.py | 8 +------- 8 files changed, 20 insertions(+), 62 deletions(-) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 3847c34df..1a2378f08 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -24,13 +24,7 @@ DecompressMemMapReader, ) -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO __all__ = ("MemoryDB", ) diff --git a/gitdb/fun.py b/gitdb/fun.py index 9baeac3ec..1d835ab35 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -21,13 +21,7 @@ except ImportError: izip = zip -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO # INVARIANTS OFS_DELTA = 6 @@ -453,7 +447,7 @@ def msb_size(data, offset=0): l = len(data) hit_msb = False while i < l: - c = ord(data[i+offset]) + c = byte_ord(data[i+offset]) size |= (c & 0x7f) << i*7 i += 1 if not c & 0x80: diff --git a/gitdb/stream.py b/gitdb/stream.py index 1bd7fe2b2..9b451506e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,13 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import BytesIO, StringIO import errno import mmap @@ -106,20 +100,20 @@ def _parse_header_info(self): self._s = maxb hdr = self.read(maxb) hdrend = hdr.find("\0".encode("ascii")) - type, size = hdr[:hdrend].split(" ".encode("ascii")) + typ, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size # adjust internal state to match actual header length that we ignore # The buffer will be depleted first on future reads self._br = 0 - hdrend += 1 # count terminating \0 - self._buf = StringIO(hdr[hdrend:]) + hdrend += 1 + self._buf = BytesIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend self._phi = True - return type, size + return typ.decode("ascii"), size #{ Interface @@ -133,8 +127,8 @@ def new(self, m, close_on_deletion=False): :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) - type, size = inst._parse_header_info() - return type, size, inst + typ, size = inst._parse_header_info() + return typ, size, inst def data(self): """:return: random access compatible data we are working on""" @@ -211,14 +205,14 @@ def read(self, size=-1): # END clamp size if size == 0: - return str() + return bytes() # END handle depletion # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream - dat = str() + dat = bytes() if self._buf: if self._buflen >= size: # have enough data @@ -588,7 +582,7 @@ class ZippedStoreShaWriter(Sha1Writer): __slots__ = ('buf', 'zip') def __init__(self): Sha1Writer.__init__(self) - self.buf = StringIO() + self.buf = BytesIO() self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) def __getattr__(self, attr): diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 18b22ff21..15fbf3fc3 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -24,13 +24,7 @@ from async import IteratorReader -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO from struct import pack @@ -41,7 +35,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world" + two_lines = "1234\nhello world".encode("ascii") all_data = (two_lines, ) def _assert_object_writing_simple(self, db): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index f52cf79d4..75342f173 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -17,13 +17,7 @@ import random from array import array -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO import glob import unittest diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index f57cc5029..c714f107f 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -8,13 +8,7 @@ from gitdb.db import LooseObjectDB from gitdb.util import pool -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO from async import IteratorReader diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f409f1788..92755d9aa 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -82,9 +82,9 @@ def test_decompress_reader(self): if with_size: # need object data zdata = zlib.compress(make_object(str_blob_type, cdata)) - type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + typ, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) - assert type == str_blob_type + assert typ == str_blob_type # even if we don't set the size, it will be set automatically on first read test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) diff --git a/gitdb/util.py b/gitdb/util.py index ed2dc3723..30c7008f4 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,13 +8,7 @@ import sys import errno -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO try: import async.mod.zlib as zlib From 0cf09d3310cba7f33b9ebc9badf61ab721d12857 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 22:15:09 -0400 Subject: [PATCH 0245/1790] Fix Python 2 failures --- gitdb/db/loose.py | 4 ++-- gitdb/db/mem.py | 4 ++-- gitdb/fun.py | 2 +- gitdb/test/db/lib.py | 11 ++++++----- gitdb/test/test_example.py | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 656841651..1b8fe643c 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -161,8 +161,8 @@ def set_ostream(self, stream): def info(self, sha): m = self._map_loose_object(sha) try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) + typ, size = loose_object_header_info(m) + return OInfo(sha, typ, size) finally: m.close() # END assure release of system resources diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1a2378f08..efd85af27 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -24,7 +24,7 @@ DecompressMemMapReader, ) -from io import StringIO +from io import BytesIO __all__ = ("MemoryDB", ) @@ -104,7 +104,7 @@ def stream_copy(self, sha_iter, odb): ostream = self.stream(sha) # compressed data including header - sio = StringIO(ostream.stream.data()) + sio = BytesIO(ostream.stream.data()) istream = IStream(ostream.type, ostream.size, sio, sha) odb.store(istream) diff --git a/gitdb/fun.py b/gitdb/fun.py index 1d835ab35..69e9826d1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -397,7 +397,7 @@ def loose_object_header_info(m): decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) - return type_name, int(size) + return type_name.decode("ascii"), int(size) def pack_object_header_info(data): """ diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 15fbf3fc3..a6cdbbe65 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -24,7 +24,7 @@ from async import IteratorReader -from io import StringIO +from io import BytesIO from struct import pack @@ -44,7 +44,7 @@ def _assert_object_writing_simple(self, db): ni = 250 for i in range(ni): data = pack(">L", i) - istream = IStream(str_blob_type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) assert new_istream is istream assert db.has_object(istream.binsha) @@ -82,7 +82,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(str_blob_type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -132,8 +132,9 @@ def _assert_object_writing_async(self, db): ni = 5000 def istream_generator(offset=0, ni=ni): for data_src in xrange(ni): - data = str(data_src + offset) - yield IStream(str_blob_type, len(data), StringIO(data)) + print(type(data_src), type(offset)) + data = bytes(data_src + offset) + yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item # END generator utility diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index c714f107f..c644b8849 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -8,7 +8,7 @@ from gitdb.db import LooseObjectDB from gitdb.util import pool -from io import StringIO +from io import BytesIO from async import IteratorReader @@ -33,8 +33,8 @@ def test_base(self): pass # END ignore exception if there are no loose objects - data = "my data" - istream = IStream("blob", len(data), StringIO(data)) + data = "my data".encode("ascii") + istream = IStream("blob", len(data), BytesIO(data)) # the object does not yet have a sha assert istream.binsha is None From 9dc111e1aa8358aa39a35d5a169335bacce53646 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 13:01:54 +0200 Subject: [PATCH 0246/1790] Added sublime-text project Suitable for everyone thanks to relative paths --- .gitignore | 2 + etc/sublime-text/gitdb.sublime-project | 54 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 etc/sublime-text/gitdb.sublime-project diff --git a/.gitignore b/.gitignore index 1e097f6f8..c6247dbb0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dist/ *.pyc *.o *.so +.noseids +*.sublime-workspace \ No newline at end of file diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project new file mode 100644 index 000000000..bc0e37f0a --- /dev/null +++ b/etc/sublime-text/gitdb.sublime-project @@ -0,0 +1,54 @@ +{ + "folders": + [ + // GITDB + //////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + "gitdb/ext" + ] + }, + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../../gitdb/ext/smmap", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + // ASYNC + //////// + { + "follow_symlinks": true, + "path": "../../gitdb/ext/async", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 1af4b42a2354acbb53c7956d647655922658fd80 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 13:05:41 +0200 Subject: [PATCH 0247/1790] Added sublime-text project Relative paths will make it work for everyone right away --- .gitignore | 2 ++ etc/sublime-text/smmap.sublime-project | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 etc/sublime-text/smmap.sublime-project diff --git a/.gitignore b/.gitignore index 73b0b6188..01247547d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ dist/ MANIFEST .tox *.egg-info +.noseids +*.sublime-workspace diff --git a/etc/sublime-text/smmap.sublime-project b/etc/sublime-text/smmap.sublime-project new file mode 100644 index 000000000..251ebbd28 --- /dev/null +++ b/etc/sublime-text/smmap.sublime-project @@ -0,0 +1,21 @@ +{ + "folders": + [ + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 0465cf327d232101b2de69d714a468b7e1a66a74 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 16 Jul 2014 20:15:31 -0400 Subject: [PATCH 0248/1790] Start up compat and encoding files There were a few things which were being reused consistently for compatibility purposes, such as the `buffer`/`memoryview` functions as well as the `izip` method which needed to be aliased for Python 3. The `buffer` function was taken from `smmap` [1] and reworked slightly to handle the optional third parameter. This also adds a compatibility file dedicated entirely to encoding issues, which seem to be the biggest problem. The main functions were taken in part from the Django project [2] and rewritten slightly because our needs are a bit more narrow. A constants file has been added to consistently handle the constants which are required for the gitdb project in the core and the tests. This is part of a greater plan to reorganize the `util.py` file included in this project. This points the async extension back at the original repository and points it to the latest commit. [1]: https://github.com/Byron/smmap/blob/1af4b42a2354acbb53c7956d647655922658fd80/smmap/util.py#L20-L26 [2]: https://github.com/django/django/blob/b8d255071ead897cf68120cd2fae7c91326ca2cc/django/utils/encoding.py --- .gitmodules | 2 +- gitdb/const.py | 5 +++++ gitdb/db/base.py | 10 ++-------- gitdb/db/loose.py | 4 +++- gitdb/ext/async | 2 +- gitdb/fun.py | 13 +++++++------ gitdb/pack.py | 18 +++++++----------- gitdb/stream.py | 24 ++++++++++++++++++------ gitdb/test/db/lib.py | 8 +++----- gitdb/util.py | 10 +++------- gitdb/utils/__init__.py | 0 gitdb/utils/compat.py | 39 +++++++++++++++++++++++++++++++++++++++ gitdb/utils/encoding.py | 35 +++++++++++++++++++++++++++++++++++ 13 files changed, 124 insertions(+), 46 deletions(-) create mode 100644 gitdb/const.py create mode 100644 gitdb/utils/__init__.py create mode 100644 gitdb/utils/compat.py create mode 100644 gitdb/utils/encoding.py diff --git a/.gitmodules b/.gitmodules index c28387570..978105388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = https://github.com/kevin-brown/async.git + url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git diff --git a/gitdb/const.py b/gitdb/const.py new file mode 100644 index 000000000..147f79cb9 --- /dev/null +++ b/gitdb/const.py @@ -0,0 +1,5 @@ +from gitdb.utils.encoding import force_bytes + +NULL_BYTE = force_bytes("\0") +NULL_HEX_SHA = "0" * 40 +NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index aa7a7ee30..53a94d249 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -178,15 +178,9 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" - if sys.version_info[0] == 3: - text_type = str - else: - text_type = basestring - - if not isinstance(rela_path, text_type): - rela_path = rela_path.decode("utf-8") + from gitdb.utils.encoding import force_text - return join(self._root_path, rela_path) + return join(self._root_path, force_text(rela_path)) #} END interface diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 1b8fe643c..f66d6275f 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -51,6 +51,8 @@ stream_copy ) +from gitdb.utils.compat import MAXSIZE + import tempfile import mmap import sys @@ -200,7 +202,7 @@ def store(self, istream): if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version - stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) + stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, diff --git a/gitdb/ext/async b/gitdb/ext/async index 339024bfb..3f26b05c2 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 339024bfb1d0a2b091e63d7a7ea23a1c63189f5c +Subproject commit 3f26b05c2f1a079d5807ed15c01b053ee846e745 diff --git a/gitdb/fun.py b/gitdb/fun.py index 69e9826d1..2749f37d0 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,10 +16,7 @@ import mmap from itertools import islice -try: - from itertools import izip -except ImportError: - izip = zip +from gitdb.utils.compat import izip from io import StringIO @@ -394,10 +391,14 @@ def loose_object_header_info(m): :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" + from gitdb.const import NULL_BYTE + from gitdb.utils.encoding import force_text + decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) - return type_name.decode("ascii"), int(size) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) + + return force_text(type_name), int(size) def pack_object_header_info(data): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index d8f32c1c2..4e83ba39d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -63,10 +63,8 @@ from binascii import crc32 -try: - from itertools import izip -except ImportError: - izip = zip +from gitdb.const import NULL_BYTE +from gitdb.utils.compat import izip, buffer import tempfile import array @@ -90,6 +88,8 @@ def pack_object_at(cursor, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" + from gitdb.utils.encoding import force_bytes + data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -120,16 +120,12 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - try: - buff = memoryview(data)[total_rela_offset:].tobytes() - except (NameError, TypeError): - buff = buffer(data, total_rela_offset) + buff = buffer(data, total_rela_offset) stream = DecompressMemMapReader(buff, False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - if hasattr(delta_info, "tobytes"): - delta_info = delta_info.tobytes() + delta_info = force_bytes(delta_info) return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: @@ -468,7 +464,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha)) + filled_sha = partial_bin_sha + NULL_BYTE * (20 - len(partial_bin_sha)) # find lowest while lo < hi: diff --git a/gitdb/stream.py b/gitdb/stream.py index 9b451506e..9e49f50e0 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -27,6 +27,10 @@ zlib ) +from gitdb.const import NULL_BYTE +from gitdb.utils.compat import buffer +from gitdb.utils.encoding import force_bytes, force_text + has_perf_mod = False try: from _perf import apply_delta as c_apply_delta @@ -99,7 +103,7 @@ def _parse_header_info(self): maxb = 512 # should really be enough, cgit uses 8192 I believe self._s = maxb hdr = self.read(maxb) - hdrend = hdr.find("\0".encode("ascii")) + hdrend = hdr.find(NULL_BYTE) typ, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size @@ -113,7 +117,7 @@ def _parse_header_info(self): self._phi = True - return typ.decode("ascii"), size + return force_text(typ), size #{ Interface @@ -264,10 +268,7 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - try: - indata = memoryview(self._m)[self._cws:self._cwe].tobytes() - except (NameError, TypeError): - indata = buffer(self._m, self._cws, self._cwe - self._cws) + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) @@ -379,6 +380,7 @@ def _set_cache_too_slow_without_c(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" + from gitdb.utils.compat import buffer # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the @@ -546,7 +548,10 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written :return: length of incoming data""" + + data = force_bytes(data) self.sha1.update(data) + return len(data) # END stream interface @@ -589,8 +594,11 @@ def __getattr__(self, attr): return getattr(self.buf, attr) def write(self, data): + data = force_bytes(data) + alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) + return alen def close(self): @@ -630,11 +638,15 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written :return: lenght of incoming data""" + data = force_bytes(data) + self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): raise self.exc + return len(data) def close(self): diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index a6cdbbe65..d0028b897 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -35,7 +35,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world".encode("ascii") + two_lines = "1234\nhello world" all_data = (two_lines, ) def _assert_object_writing_simple(self, db): @@ -81,8 +81,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - - istream = IStream(str_blob_type, len(data), BytesIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii"))) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -131,8 +130,7 @@ def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - print(type(data_src), type(offset)) + for data_src in range(ni): data = bytes(data_src + offset) yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item diff --git a/gitdb/util.py b/gitdb/util.py index 30c7008f4..5a82c553e 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -30,10 +30,7 @@ mman = SlidingWindowMapManager() #END handle mman -try: - import hashlib -except ImportError: - import sha +import hashlib try: from struct import unpack_from @@ -84,9 +81,8 @@ def unpack_from(fmt, data, offset=0): close = os.close fsync = os.fsync -# constants -NULL_HEX_SHA = "0"*40 -NULL_BIN_SHA = "\0"*20 +# Backwards compatibility imports +from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA #} END Aliases diff --git a/gitdb/utils/__init__.py b/gitdb/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py new file mode 100644 index 000000000..b9da683fa --- /dev/null +++ b/gitdb/utils/compat.py @@ -0,0 +1,39 @@ +import sys + +PY3 = sys.version_info[0] == 3 + +try: + from itertools import izip +except ImportError: + izip = zip + +try: + # Python 2 + buffer = buffer + memoryview = buffer +except NameError: + # Python 3 has no `buffer`; only `memoryview` + def buffer(obj, offset, size=None): + if size is None: + return memoryview(obj)[offset:] + else: + return memoryview(obj[offset:offset+size]) + + memoryview = memoryview + +if PY3: + MAXSIZE = sys.maxsize +else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py new file mode 100644 index 000000000..12164e756 --- /dev/null +++ b/gitdb/utils/encoding.py @@ -0,0 +1,35 @@ +from gitdb.utils import compat + +if compat.PY3: + string_types = (str, ) + text_type = str +else: + string_types = (basestring, ) + text_type = unicode + +def force_bytes(data, encoding="utf-8"): + if isinstance(data, bytes): + return data + + if isinstance(data, compat.memoryview): + return bytes(data) + + if isinstance(data, string_types): + return data.encode(encoding) + + return data + +def force_text(data, encoding="utf-8"): + if isinstance(data, text_type): + return data + + if isinstance(data, string_types): + return data.decode(encoding) + + if not isinstance(data, bytes): + data = force_bytes(data, encoding) + + if compat.PY3: + return text_type(data, encoding) + else: + return text_type(data) From ecdae96cdb8bbde322f59fd119dab302e7797c18 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 16 Jul 2014 20:36:02 -0400 Subject: [PATCH 0249/1790] Fixed a few more encoding issues Bytes should always be returned from the streams, so the tests should be checking against byte strings instead of text strings. This also fixes the `sha_iter` as it relied on the Python 2 `iterkeys` which has been renamed to `keys` in Python 3. --- gitdb/db/loose.py | 3 ++- gitdb/db/mem.py | 5 ++++- gitdb/test/db/lib.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index f66d6275f..63f96352b 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -52,6 +52,7 @@ ) from gitdb.utils.compat import MAXSIZE +from gitdb.utils.encoding import force_bytes import tempfile import mmap @@ -116,7 +117,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha): :raise BadObject: """ candidate = None for binsha in self.sha_iter(): - if bin_to_hex(binsha).startswith(partial_hexsha): + if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)): # it can't ever find the same object twice if candidate is not None: raise AmbiguousObjectName(partial_hexsha) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index efd85af27..a22454685 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -86,7 +86,10 @@ def size(self): return len(self._cache) def sha_iter(self): - return self._cache.iterkeys() + try: + return self._cache.iterkeys() + except AttributeError: + return self._cache.keys() #{ Interface diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index d0028b897..38747deb2 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -21,6 +21,7 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type +from gitdb.utils.encoding import force_bytes from async import IteratorReader @@ -97,7 +98,7 @@ def _assert_object_writing(self, db): assert info.size == len(data) ostream = db.stream(sha) - assert ostream.read() == data + assert ostream.read() == force_bytes(data) assert ostream.type == str_blob_type assert ostream.size == len(data) else: From c63dcacbfa996b1d0d81d50c359fa37e4906cfb1 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Mon, 21 Jul 2014 20:38:17 -0400 Subject: [PATCH 0250/1790] Convert types to bytes This makes it easier to deal with things internally as now everything is passed as bytes. --- gitdb/fun.py | 32 +++++++++++++++++++------------- gitdb/stream.py | 4 ++-- gitdb/typ.py | 12 +++++------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 2749f37d0..6495359bc 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -17,6 +17,12 @@ from itertools import islice from gitdb.utils.compat import izip +from gitdb.typ import ( + str_blob_type, + str_commit_type, + str_tree_type, + str_tag_type, +) from io import StringIO @@ -27,23 +33,23 @@ type_id_to_type_map = { 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", + 1 : str_commit_type, + 2 : str_tree_type, + 3 : str_blob_type, + 4 : str_tag_type, 5 : "", # EXT 2 OFS_DELTA : "OFS_DELTA", # OFFSET DELTA REF_DELTA : "REF_DELTA" # REFERENCE DELTA } -type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA -) +type_to_type_id_map = { + str_commit_type: 1, + str_tree_type: 2, + str_blob_type: 3, + str_tag_type: 4, + "OFS_DELTA": OFS_DELTA, + "REF_DELTA": REF_DELTA, +} # used when dealing with larger streams chunk_size = 1000 * mmap.PAGESIZE @@ -398,7 +404,7 @@ def loose_object_header_info(m): hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) - return force_text(type_name), int(size) + return type_name, int(size) def pack_object_header_info(data): """ diff --git a/gitdb/stream.py b/gitdb/stream.py index 9e49f50e0..5dd38ec97 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -29,7 +29,7 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import buffer -from gitdb.utils.encoding import force_bytes, force_text +from gitdb.utils.encoding import force_bytes has_perf_mod = False try: @@ -117,7 +117,7 @@ def _parse_header_info(self): self._phi = True - return force_text(typ), size + return typ, size #{ Interface diff --git a/gitdb/typ.py b/gitdb/typ.py index e84dd2455..edd1f2731 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,11 +4,9 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -#{ String types +from gitdb.utils.encoding import force_bytes -str_blob_type = "blob" -str_commit_type = "commit" -str_tree_type = "tree" -str_tag_type = "tag" - -#} END string types +str_blob_type = force_bytes("blob") +str_commit_type = force_bytes("commit") +str_tree_type = force_bytes("tree") +str_tag_type = force_bytes("tag") From fda32852a1dd6f6b0988248fb2c7921104e3ea6c Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Fri, 25 Jul 2014 21:12:33 +0200 Subject: [PATCH 0251/1790] warnings fixes --- setup.py | 2 ++ smmap/test/test_buf.py | 44 ++++++++++++----------- smmap/test/test_mman.py | 79 ++++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/setup.py b/setup.py index 13d2063ab..204a2a75d 100644 --- a/setup.py +++ b/setup.py @@ -52,4 +52,6 @@ "Programming Language :: Python :: 3.4", ], long_description=long_description, + tests_require=('nose', 'nosexcover'), + test_suite='nose.collector' ) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 15dfb8238..807d2770f 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,3 +1,5 @@ +from __future__ import with_statement, print_function + from .lib import TestBase, FileCreator from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager @@ -17,65 +19,66 @@ static_man = StaticWindowMapManager() class TestBuf(TestBase): - + def test_basics(self): fc = FileCreator(self.k_window_test_size, "buffer_test") - + # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - + buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None - + # can call end access any time buf.end_access() buf.end_access() assert len(buf) == 0 - + # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True assert len(buf) == fc.size - offset assert buf.cursor().is_valid() - + # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True assert len(buf) == fc.size assert buf.cursor().is_valid() - + # simple access - data = open(fc.path, 'rb').read() + with open(fc.path, 'rb') as fp: + data = fp.read() assert data[offset] == buf[0] assert data[offset:offset*2] == buf[0:offset] - + # negative indices, partial slices assert buf[-1] == buf[len(buf)-1] assert buf[-10:] == buf[len(buf)-10:len(buf)] - + # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() assert buf.cursor().is_associated() # but it remains associated - + # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically del(c) - + assert man_optimal.num_file_handles() == 1 - + # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to + # blast away with rnadom access and a full mapping - we don't want to # exagerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which + # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), + for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case'), (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) @@ -84,7 +87,7 @@ def test_basics(self): num_accesses_left = max_num_accesses num_bytes = 0 fsize = fc.size - + st = time() buf.begin_access() while num_accesses_left: @@ -102,7 +105,7 @@ def test_basics(self): num_bytes += 1 #END handle mode # END handle num accesses - + buf.end_access() assert manager.num_file_handles() assert manager.collect() @@ -110,8 +113,9 @@ def test_basics(self): elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed), + file=sys.stderr) # END handle access mode # END for each manager # END for each input diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e0516b21c..4d1839eca 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,3 +1,5 @@ +from __future__ import with_statement, print_function + from .lib import TestBase, FileCreator from smmap.mman import * @@ -12,43 +14,43 @@ from copy import copy class TestMMan(TestBase): - + def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") - + man = SlidingWindowMapManager() ci = WindowCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state - + cv = man.make_cursor(fc.path) assert not cv.is_valid() # no region mapped yet assert cv.is_associated()# but it know where to map it from assert cv.file_size() == fc.size assert cv.path() == fc.path - + # copy module cio = copy(cv) assert not cio.is_valid() and cio.is_associated() - + # assign method assert not ci.is_associated() ci.assign(cv) assert not ci.is_valid() and ci.is_associated() - + # unuse non-existing region is fine cv.unuse_region() cv.unuse_region() - + # destruction is fine (even multiple times) cv._destroy() WindowCursor(man)._destroy() - + def test_memory_manager(self): slide_man = SlidingWindowMapManager() static_man = StaticWindowMapManager() - + for man in (static_man, slide_man): assert man.num_file_handles() == 0 assert man.num_open_files() == 0 @@ -59,15 +61,15 @@ def test_memory_manager(self): assert man.window_size() > winsize_cmp_val assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 - + # collection doesn't raise in 'any' mode man._collect_lru_region(0) # doesn't raise if we are within the limit man._collect_lru_region(10) - - # doesn't fail if we overallocate + + # doesn't fail if we overallocate assert man._collect_lru_region(sys.maxsize) == 0 - + # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") fd = os.open(fc.path, os.O_RDONLY) @@ -77,8 +79,9 @@ def test_memory_manager(self): assert c.use_region(10, 10).is_valid() assert c.ofs_begin() == 10 assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] - + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + if isinstance(item, int): self.assertRaises(ValueError, c.path) else: @@ -87,38 +90,39 @@ def test_memory_manager(self): #END for each input os.close(fd) # END for each manager type - + def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") - data = open(fc.path, 'rb').read() + with open(fc.path, 'rb') as fp: + data = fp.read() fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 - #small_size = + #small_size = for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size - + # small windows, a reasonable max memory. Not too many regions at once man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) c = man.make_cursor(item) - + # still empty (more about that is tested in test_memory_manager() assert man.num_open_files() == 0 assert man.mapped_memory_size() == 0 - + base_offset = 5000 # window size is 0 for static managers, hence size will be 0. We take that into consideration size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() assert rr().client_count() == 2 # the manager and the cursor and us - + assert man.num_open_files() == 1 assert man.num_file_handles() == 1 assert man.mapped_memory_size() == rr().size() - + #assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded @@ -127,9 +131,9 @@ def test_memman_operation(self): else: assert rr().size() == fc.size #END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 assert c.use_region(0, nsize).is_valid() @@ -138,7 +142,7 @@ def test_memman_operation(self): assert c.size() == nsize assert c.ofs_begin() == 0 assert c.buffer()[:] == data[:nsize] - + # map some part at the end, our requested size cannot be kept overshoot = 4000 base_offset = fc.size - (size or c.size()) + overshoot @@ -156,23 +160,23 @@ def test_memman_operation(self): assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left assert rr().ofs_end() <= fc.size # it cannot be larger than the file assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - + # unising a region makes the cursor invalid c.unuse_region() assert not c.is_valid() if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only + # but doesn't change anything regarding the handle count - we cache it and only # remove mapped regions if we have to assert man.num_file_handles() == 2 #END ignore this for static managers - + # iterate through the windows, verify data contents # this will trigger map collection after a while max_random_accesses = 5000 num_random_accesses = max_random_accesses memory_read = 0 st = time() - + # cache everything to get some more performance includes_ofs = c.includes_ofs max_mapped_memory_size = man.max_mapped_memory_size() @@ -182,7 +186,7 @@ def test_memman_operation(self): while num_random_accesses: num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) - + # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() @@ -192,19 +196,20 @@ def test_memman_operation(self): csize = c.size() assert c.buffer()[:] == data[base_offset:base_offset+csize] memory_read += csize - + assert includes_ofs(base_offset) assert includes_ofs(base_offset+csize-1) assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) - sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed), + file=sys.stderr) + # an offset as large as the size doesn't work ! assert not c.use_region(fc.size, size).is_valid() - + # collection - it should be able to collect all assert man.num_file_handles() assert man.collect() From cc32b0a99744b07bcacf3b027af597a4bdd10b85 Mon Sep 17 00:00:00 2001 From: Orgad Shaneh Date: Wed, 10 Sep 2014 09:51:54 +0300 Subject: [PATCH 0252/1790] Setup: Fix invalid syntax --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62bc6d007..07f810be8 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def run(self): try: build_ext.run(self) except Exception: - print "Ignored failure when building extensions, pure python modules will be used instead" + print("Ignored failure when building extensions, pure python modules will be used instead") # END ignore errors From a9f7f7bc5a2819924992dcb536c8e72399ae6f16 Mon Sep 17 00:00:00 2001 From: Matt Hickford Date: Mon, 6 Oct 2014 17:35:19 +0100 Subject: [PATCH 0253/1790] Add installation instructions to readme Also, fix broken badge link. --- README.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0fc0534e6..f97d5c31e 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,20 @@ aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to operate on large objects with a small memory footprint. +Installation +============ + +.. image:: https://pypip.in/version/gitdb/badge.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Latest Version +.. image:: https://pypip.in/py_versions/gitdb/badge.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Supported Python versions + +From `PyPI `_ + + pip install gitdb + REQUIREMENTS ============ @@ -33,8 +47,9 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - +.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master + :target: https://travis-ci.org/gitpython-developers/gitdb + https://github.com/gitpython-developers/gitdb/issues LICENSE From 0e0b7e253f7635af9e24a15f4393481495565667 Mon Sep 17 00:00:00 2001 From: Matt Hickford Date: Mon, 6 Oct 2014 17:39:18 +0100 Subject: [PATCH 0254/1790] Clarify which Python versions are supported --- setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 07f810be8..6dd4fa4b4 100755 --- a/setup.py +++ b/setup.py @@ -90,5 +90,13 @@ def get_data_files(self): zip_safe=False, requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), - long_description = """GitDB is a pure-Python git object database""" + long_description = """GitDB is a pure-Python git object database""", + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], ) From ef59e98f74eeb00373b02f7a2b8bed1943924de9 Mon Sep 17 00:00:00 2001 From: David Schryer Date: Wed, 5 Nov 2014 10:48:33 +0200 Subject: [PATCH 0255/1790] Minor change to begin Python3 compatibility support. --- gitdb/db/ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 60004a77a..0a28b9efb 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -68,7 +68,7 @@ def _update_dbs_from_ref_file(self): db.databases() # END verification self._dbs.append(db) - except Exception, e: + except Exception as e: # ignore invalid paths or issues pass # END for each path to add From 977e666e2489ddc669a06481bb5192b59854da8d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 12 Nov 2014 20:30:47 +0100 Subject: [PATCH 0256/1790] Initial improvements to get rid of the performance regression in py3. Byte buffer concatenations are considerably slower here for some reason. Also there was no need for the memorybuffer. --- .travis.yml | 2 ++ README.md | 7 +------ doc/source/changes.rst | 6 +++++- smmap/buf.py | 10 +++++----- smmap/mman.py | 29 ++++++++++++++--------------- smmap/util.py | 5 +++-- 6 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47cb41170..1f7872c86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python python: + - 2.4 + - 2.5 - 2.6 - 2.7 - 3.3 diff --git a/README.md b/README.md index c056ed1c1..6d2353820 100644 --- a/README.md +++ b/README.md @@ -25,19 +25,15 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.4, 2.5 or 2.6 +* Python 2.4, 2.5, 2.6, 2.7 or 3.3 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. - - ## Limitations * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. -* It wasn't tested on python 2.7 and 3.x. - ## Installing smmap @@ -80,7 +76,6 @@ Issues can be filed on github: * https://github.com/Byron/smmap/issues - ## License Information *smmap* is licensed under the New BSD License. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 03148fb31..6174008e9 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,12 +2,16 @@ Changelog ######### +********** +v0.8.2 +********** +- Cleaned up code and assured it works sufficiently well with python 3 + ********** v0.8.1 ********** - A single bugfix - ********** v0.8.0 ********** diff --git a/smmap/buf.py b/smmap/buf.py index 2f27d4d01..4cde6dde4 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -79,18 +79,18 @@ def __getslice__(self, i, j): else: l = j-i # total length ofs = i - # Keeping tokens in a list could possible be faster, but the list - # overhead outweighs the benefits (tested) ! - md = bytes() + # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower + # in the previous iteration of this code + md = list() while l: c.use_region(ofs, l) assert c.is_valid() d = c.buffer()[:l] ofs += len(d) l -= len(d) - md += d + md.append(d) #END while there are bytes to read - return md + return bytes().join(md) # END fast or slow path #{ Interface diff --git a/smmap/mman.py b/smmap/mman.py index 7cbb535bd..a89efd474 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,13 +1,12 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - align_to_mmap, - string_types, - buffer, - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + string_types, + buffer, + ) from weakref import ref import sys @@ -261,14 +260,14 @@ class StaticWindowMapManager(object): def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): """initialize the manager with the given parameters. :param window_size: if -1, a default window size will be chosen depending on - the operating system's architechture. It will internally be quantified to a multiple of the page size + the operating system's architecture. It will internally be quantified to a multiple of the page size If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. - If 0, a viable default iwll be set dependning on the system's architecture. + If 0, a viable default will be set depending on the system's architecture. It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate - :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, - the manager will free as many handles as posisble""" + :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + the manager will free as many handles as possible""" self._fdict = dict() self._window_size = window_size self._max_memory_size = max_memory_size @@ -277,7 +276,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._handle_count = 0 if window_size < 0: - coeff = 32 + coeff = 64 if is_64_bit(): coeff = 1024 #END handle arch @@ -285,7 +284,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. # END handle max window size if max_memory_size == 0: - coeff = 512 + coeff = 1024 if is_64_bit(): coeff = 8192 #END handle arch diff --git a/smmap/util.py b/smmap/util.py index c37dfdd31..3396d90c0 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,8 +23,9 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - return memoryview(obj[offset:offset+size]) - + # return memoryview(obj[offset:offset+size]) + # doing it directly is much faster ! + return obj[offset:offset+size] def string_types(): if sys.version_info[0] >= 3: From 948a9274527d14702875581d7115389cf9aa8244 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 08:21:09 +0100 Subject: [PATCH 0257/1790] Fixed a few typos and major linter errors --- .travis.yml | 5 +++-- README.md | 4 +--- doc/source/changes.rst | 2 +- doc/source/intro.rst | 8 ++------ setup.py | 10 +++++----- smmap/__init__.py | 2 +- smmap/buf.py | 2 -- smmap/mman.py | 16 ++++++---------- smmap/test/test_buf.py | 14 +++++++++----- smmap/test/test_mman.py | 12 +++++++----- smmap/test/test_util.py | 9 ++++++++- smmap/util.py | 4 ++-- 12 files changed, 45 insertions(+), 43 deletions(-) mode change 100644 => 100755 setup.py diff --git a/.travis.yml b/.travis.yml index 1f7872c86..cb0c16e44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: python python: - - 2.4 - - 2.5 + # These versions are unsupported by travis, even though smmap claims to still support these outdated versions + # - 2.4 + # - 2.5 - 2.6 - 2.7 - 3.3 diff --git a/README.md b/README.md index 6d2353820..327d66372 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,9 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively: +Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: ```bash -$ easy_install smmap -# or $ pip install smmap ``` diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6174008e9..d5ed8e378 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ********** -v0.8.2 +v0.8.3 ********** - Cleaned up code and assured it works sufficiently well with python 3 diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 30bff0ded..ee3108a6a 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.4, 2.5 or 2.6 +* Python 2.4, 2.5, 2.6, 2.7 or 3.3 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. @@ -32,15 +32,12 @@ Limitations ########### * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. -* It wasn't tested on python 2.7 and 3.x. ################ Installing smmap ################ -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: +Its easiest to install smmap using the *pip* program:: - $ easy_install smmap - # or $ pip install smmap As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. @@ -75,5 +72,4 @@ License Information ################### *smmap* is licensed under the New BSD License. -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools .. _pip: http://www.pip-installer.org/en/latest/ diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 204a2a75d..c6afc8960 --- a/setup.py +++ b/setup.py @@ -10,10 +10,10 @@ import smmap -if os.path.exists("README.rst"): - long_description = codecs.open('README.rst', "r", "utf-8").read() +if os.path.exists("README.md"): + long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/nvie/smmap/tree/master" + long_description = "See http://github.com/Byron/smmap" setup( name="smmap", @@ -32,8 +32,8 @@ #"Development Status :: 1 - Planning", #"Development Status :: 2 - Pre-Alpha", #"Development Status :: 3 - Alpha", - "Development Status :: 4 - Beta", - #"Development Status :: 5 - Production/Stable", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", #"Development Status :: 6 - Mature", #"Development Status :: 7 - Inactive", "Environment :: Console", diff --git a/smmap/__init__.py b/smmap/__init__.py index 879ebea24..c494648d7 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 2) +version_info = (0, 8, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index 4cde6dde4..ef9d49e46 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,6 +1,4 @@ """Module with a simple buffer implementation using the memory manager""" -from .mman import WindowCursor - import sys __all__ = ["SlidingWindowMapBuffer"] diff --git a/smmap/mman.py b/smmap/mman.py index a89efd474..da6fd8153 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -101,7 +101,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. - This is not the case if the mapping failed becaues we reached the end of the file + This is not the case if the mapping failed because we reached the end of the file **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" @@ -137,7 +137,7 @@ def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it + to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None # note: should reset ofs and size, but we spare that for performance. Its not @@ -203,7 +203,7 @@ def file_size(self): return self._rlist.file_size() def path_or_fd(self): - """:return: path or file decriptor of the underlying mapped file""" + """:return: path or file descriptor of the underlying mapped file""" return self._rlist.path_or_fd() def path(self): @@ -237,12 +237,12 @@ class StaticWindowMapManager(object): These clients would have to use a SlidingWindowMapBuffer to hide this fact. This type will always use a maximum window size, and optimize certain methods to - acomodate this fact""" + accommodate this fact""" __slots__ = [ '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_memory_size', # maximum amount of memory we may allocate '_max_handle_count', # maximum amount of handles to keep open '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles @@ -264,7 +264,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default will be set depending on the system's architecture. - It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate + It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, the manager will free as many handles as possible""" @@ -350,8 +350,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # As many more operations are likely to fail in that condition ( # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway if is_recursive: # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it @@ -562,8 +560,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # As many more operations are likely to fail in that condition ( # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway if is_recursive: # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 807d2770f..d3e51e2ee 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,14 +1,18 @@ -from __future__ import with_statement, print_function +from __future__ import print_function from .lib import TestBase, FileCreator -from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager -from smmap.buf import * +from smmap.mman import ( + SlidingWindowMapManager, + StaticWindowMapManager + ) +from smmap.buf import SlidingWindowMapBuffer from random import randint from time import time import sys import os +import logging man_optimal = SlidingWindowMapManager() @@ -71,8 +75,8 @@ def test_basics(self): assert man_optimal.num_file_handles() == 1 # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to - # exagerate the manager's overhead, but measure the buffer overhead + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 4d1839eca..cc5d91488 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,11 +1,13 @@ -from __future__ import with_statement, print_function +from __future__ import print_function from .lib import TestBase, FileCreator -from smmap.mman import * -from smmap.mman import WindowCursor +from smmap.mman import ( + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager + ) from smmap.util import align_to_mmap -from smmap.exc import RegionCollectionError from random import randint from time import time @@ -67,7 +69,7 @@ def test_memory_manager(self): # doesn't raise if we are within the limit man._collect_lru_region(10) - # doesn't fail if we overallocate + # doesn't fail if we over-allocate assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 8afba005e..745da83d7 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,6 +1,13 @@ from .lib import TestBase, FileCreator -from smmap.util import * +from smmap.util import ( + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap + ) import os import sys diff --git a/smmap/util.py b/smmap/util.py index 3396d90c0..a4d7d8f1d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -113,7 +113,7 @@ class MapRegion(object): '__weakref__' ] _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 - + if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot @@ -283,4 +283,4 @@ def file_size(self): #END update file size return self._file_size -#} END utilty classes +#} END utility classes From 85dde34bc724570617f3df1cdc40ba1b0942c77e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 09:00:44 +0100 Subject: [PATCH 0258/1790] Minor adjustments to adapt to changes in async (due to be removed anyway) --- gitdb/__init__.py | 2 +- gitdb/base.py | 6 +----- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 2 +- gitdb/pack.py | 3 ++- gitdb/stream.py | 3 +-- gitdb/test/lib.py | 2 -- gitdb/test/test_stream.py | 3 +-- gitdb/util.py | 23 +++++++++-------------- setup.py | 35 ++++++++++++++++++++++++++--------- 11 files changed, 44 insertions(+), 39 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ff750d14c..bb1d13b47 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 4) +version_info = (0, 5, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/base.py b/gitdb/base.py index bad5f7472..3476d0d1a 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -3,11 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" -from util import ( - bin_to_hex, - zlib - ) - +from util import bin_to_hex from fun import ( type_id_to_type_map, type_to_type_id_map diff --git a/gitdb/ext/async b/gitdb/ext/async index 90326fb86..b930ee15c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 +Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 616e9ceaf..f53ddc686 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd +Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf diff --git a/gitdb/fun.py b/gitdb/fun.py index c1e73e895..177a2fb56 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -10,7 +10,7 @@ BadObjectType ) -from util import zlib +import zlib decompressobj = zlib.decompressobj import mmap diff --git a/gitdb/pack.py b/gitdb/pack.py index 48121f026..e2673ee1b 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -3,13 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" +import zlib + from gitdb.exc import ( BadObject, UnsupportedOperation, ParseError ) from util import ( - zlib, mman, LazyMixin, unpack_from, diff --git a/gitdb/stream.py b/gitdb/stream.py index 6441b1e1a..cbb10c1f1 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from cStringIO import StringIO -import errno +import zlib import mmap import os @@ -23,7 +23,6 @@ make_sha, write, close, - zlib ) has_perf_mod = False diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ac8473a4e..af57e46d1 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -11,8 +11,6 @@ ZippedStoreShaWriter ) -from gitdb.util import zlib - import sys import random from array import array diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 6dc27463c..d2487f6d1 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -18,12 +18,11 @@ hex_to_bin ) -from gitdb.util import zlib +import zlib from gitdb.typ import ( str_blob_type ) -import time import tempfile import os diff --git a/gitdb/util.py b/gitdb/util.py index 1662b662d..75f6bb927 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -7,27 +7,22 @@ import mmap import sys import errno - -from cStringIO import StringIO +import stat # in py 2.4, StringIO is only StringI, without write support. # Hence we must use the python implementation for this if sys.version_info[1] < 5: from StringIO import StringIO +else: + from cStringIO import StringIO # END handle python 2.4 -try: - import async.mod.zlib as zlib -except ImportError: - import zlib -# END try async zlib - from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer + ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -304,7 +299,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode, 0600) + fd = os.open(self._lockfilepath(), lockmode, stat.S_IREAD|stat.S_IWRITE) if not write: os.close(fd) else: @@ -373,7 +368,7 @@ def _end_writing(self, successful=True): # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well - chmod(self._filepath, 0644) + chmod(self._filepath, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) else: # just delete the file so far, we failed os.remove(lockfile) diff --git a/setup.py b/setup.py index 6dd4fa4b4..639370471 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 4) +version_info = (0, 5, 5) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, @@ -88,15 +88,32 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), + requires=('async (>=0.6.2)', 'smmap (>=0.8.3)'), install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - ], - ) + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + ],) From a8f2f63823324ad76cbb36b0f4115e73c7d9d594 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 10:31:45 +0100 Subject: [PATCH 0259/1790] Made sure xrange is used instead of range in python 2 range in py2 will return a list, which can mean a lot of time and memory is spent on generating it even though it's just used for iteration. Simplified implementation of MAXSIZE --- gitdb/db/pack.py | 3 ++- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/pack.py | 10 +++++----- gitdb/test/db/lib.py | 5 +++-- gitdb/test/lib.py | 3 ++- gitdb/test/test_pack.py | 3 ++- gitdb/util.py | 5 +---- gitdb/utils/compat.py | 20 +++++--------------- 9 files changed, 22 insertions(+), 31 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eca02bbff..b95bfed9d 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -18,6 +18,7 @@ ) from gitdb.pack import PackEntity +from gitdb.utils.compat import xrange from functools import reduce @@ -106,7 +107,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in range(index.size()): + for index in xrange(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/ext/async b/gitdb/ext/async index 3f26b05c2..b930ee15c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 3f26b05c2f1a079d5807ed15c01b053ee846e745 +Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 552671191..f53ddc686 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 55267119140f3828a24b4986600ed21a1808d6cc +Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf diff --git a/gitdb/pack.py b/gitdb/pack.py index 4e83ba39d..0b3ffd53e 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -64,7 +64,7 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import izip, buffer +from gitdb.utils.compat import izip, buffer, xrange import tempfile import array @@ -208,7 +208,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in range(255): + for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i+1] += v @@ -374,7 +374,7 @@ def _read_fanout(self, byte_offset): d = self._cursor.map() out = list() append = out.append - for i in range(256): + for i in xrange(256): append(unpack_from('>L', d, byte_offset + i*4)[0]) # END for each entry return out @@ -415,7 +415,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in range(self.size())) + return tuple(self.offset(index) for index in xrange(self.size())) # END handle version def sha_to_index(self, sha): @@ -703,7 +703,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in range(self._index.size()): + for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 38747deb2..8e333ddf2 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -22,6 +22,7 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type from gitdb.utils.encoding import force_bytes +from gitdb.utils.compat import xrange from async import IteratorReader @@ -43,7 +44,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in range(ni): + for i in xrange(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) @@ -131,7 +132,7 @@ def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 def istream_generator(offset=0, ni=ni): - for data_src in range(ni): + for data_src in xrange(ni): data = bytes(data_src + offset) yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 75342f173..d692224ff 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -12,6 +12,7 @@ ) from gitdb.util import zlib +from gitdb.utils.compat import xrange import sys import random @@ -110,7 +111,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes // 4 - producer = range(actual_size) + producer = xrange(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index bcda3cfb8..6b67ad887 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -26,6 +26,7 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha +from gitdb.utils.compat import xrange from itertools import chain try: @@ -64,7 +65,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in range(index.size()): + for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/util.py b/gitdb/util.py index 5a82c553e..a3c44d446 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -122,13 +122,10 @@ def byte_ord(b): #{ Routines -def make_sha(source=None): +def make_sha(source=''.encode("ascii")): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ - if source is None: - source = "".encode("ascii") - try: return hashlib.sha1(source) except NameError: diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index b9da683fa..eec319f3d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,8 +4,10 @@ try: from itertools import izip + xrange = xrange except ImportError: izip = zip + xrange = range try: # Python 2 @@ -21,19 +23,7 @@ def buffer(obj, offset, size=None): memoryview = memoryview -if PY3: +try: + MAXSIZE = sys.maxint +except AttributeError: MAXSIZE = sys.maxsize -else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X From 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 11:28:27 +0100 Subject: [PATCH 0260/1790] Fixed incorrect usage of memoryview. It's not getting faster though [ skip ci ] --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index a4d7d8f1d..44e94124d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,7 +23,7 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - # return memoryview(obj[offset:offset+size]) + # return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! return obj[offset:offset+size] From b64c771bcb2ec336dd549cfe9d072340c886f3c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 12:17:01 +0100 Subject: [PATCH 0261/1790] Fixed all applicable lint issues --- gitdb/db/base.py | 2 -- gitdb/db/git.py | 7 +------ gitdb/db/loose.py | 8 -------- gitdb/db/pack.py | 1 - gitdb/db/ref.py | 1 - gitdb/ext/smmap | 2 +- gitdb/fun.py | 11 +++-------- gitdb/pack.py | 7 ++----- gitdb/stream.py | 4 +--- gitdb/test/db/lib.py | 8 ++++++-- gitdb/test/db/test_git.py | 6 +++++- gitdb/test/db/test_loose.py | 5 ++++- gitdb/test/db/test_mem.py | 5 ++++- gitdb/test/db/test_pack.py | 11 +++++++---- gitdb/test/lib.py | 9 +-------- gitdb/test/performance/lib.py | 4 +--- gitdb/test/performance/test_pack.py | 2 +- gitdb/test/performance/test_pack_streaming.py | 1 - gitdb/test/performance/test_stream.py | 18 +++++++----------- gitdb/test/test_base.py | 10 +++++++++- gitdb/test/test_example.py | 5 ++++- gitdb/test/test_pack.py | 3 --- gitdb/test/test_stream.py | 11 ++++++----- gitdb/test/test_util.py | 1 - gitdb/util.py | 6 +++++- gitdb/utils/compat.py | 10 ++++++++-- 26 files changed, 76 insertions(+), 82 deletions(-) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 53a94d249..c534705b7 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,7 +22,6 @@ from itertools import chain from functools import reduce -import sys __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -206,7 +205,6 @@ def _databases_recursive(database, output): """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" if isinstance(database, CompoundDB): - compounds = list() dbs = database.databases() output.extend(db for db in dbs if not isinstance(db, CompoundDB)) for cdb in (db for db in dbs if isinstance(db, CompoundDB)): diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 5c74a2049..d22e3f1b9 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -12,12 +12,7 @@ from gitdb.db.pack import PackedDB from gitdb.db.ref import ReferenceDB -from gitdb.util import LazyMixin -from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName -) +from gitdb.exc import InvalidDBRoot import os diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 63f96352b..3abdaa96f 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -10,7 +10,6 @@ from gitdb.exc import ( - InvalidDBRoot, BadObject, AmbiguousObjectName ) @@ -55,8 +54,6 @@ from gitdb.utils.encoding import force_bytes import tempfile -import mmap -import sys import os @@ -149,11 +146,6 @@ def _map_loose_object(self, sha): raise BadObject(sha) # END handle error # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index b95bfed9d..4d0a7f8b3 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -103,7 +103,6 @@ def stream(self, sha): return entity.stream_at_index(index) def sha_iter(self): - sha_list = list() for entity in self.entities(): index = entity.index() sha_by_index = index.sha diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 748f7c145..d98912643 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -6,7 +6,6 @@ CompoundDB, ) -import os __all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f53ddc686..28fd45e0a 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf +Subproject commit 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 diff --git a/gitdb/fun.py b/gitdb/fun.py index 22e56ddbb..80f01e6b2 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -6,18 +6,15 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from gitdb.exc import ( - BadObjectType -) - import zlib from gitdb.util import byte_ord decompressobj = zlib.decompressobj import mmap from itertools import islice +from functools import reduce -from gitdb.utils.compat import izip +from gitdb.utils.compat import izip, buffer, xrange from gitdb.typ import ( str_blob_type, str_commit_type, @@ -247,7 +244,6 @@ def compress(self): if slen < 2: return self i = 0 - slen_orig = slen first_data_index = None while i < slen: @@ -399,7 +395,6 @@ def loose_object_header_info(m): object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" from gitdb.const import NULL_BYTE - from gitdb.utils.encoding import force_text decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) @@ -684,7 +679,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - # raise ImportError; # DEBUG + # NOQA from _perf import connect_deltas except ImportError: pass diff --git a/gitdb/pack.py b/gitdb/pack.py index fbfd18b08..87818b6e7 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -7,6 +7,7 @@ from gitdb.exc import ( BadObject, + AmbiguousObjectName, UnsupportedOperation, ParseError ) @@ -57,11 +58,7 @@ FlexibleSha1Writer ) -from struct import ( - pack, - unpack, -) - +from struct import pack from binascii import crc32 from gitdb.const import NULL_BYTE diff --git a/gitdb/stream.py b/gitdb/stream.py index de5884859..5daf01cb6 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,9 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from io import BytesIO, StringIO +from io import BytesIO -import errno import mmap import os import zlib @@ -15,7 +14,6 @@ stream_copy, apply_delta_data, connect_deltas, - DeltaChunkList, delta_types ) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 8e333ddf2..67958c366 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -6,12 +6,16 @@ from gitdb.test.lib import ( with_rw_directory, with_packs_rw, - ZippedStoreShaWriter, fixture_path, TestBase ) -from gitdb.stream import Sha1Writer + + +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) from gitdb.base import ( IStream, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index cce2b9c09..4bce7dac8 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,7 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + fixture_path, + with_rw_directory +) from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 5e42b639a..1299f7b86 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -2,7 +2,10 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 9235b21d3..97f721719 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -2,7 +2,10 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) from gitdb.db import ( MemoryDB, LooseObjectDB diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f5a4dcb92..1177cf962 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -2,9 +2,12 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + with_packs_rw +) from gitdb.db import PackedDB -from gitdb.test.lib import fixture_path from gitdb.exc import BadObject, AmbiguousObjectName @@ -46,8 +49,8 @@ def test_writing(self, path): random.shuffle(sha_list) for sha in sha_list: - info = pdb.info(sha) - stream = pdb.stream(sha) + pdb.info(sha) + pdb.stream(sha) # END for each sha to query diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index cb77e69bf..d88ec8b43 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -3,14 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" -from gitdb import ( - OStream, - ) -from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter -) - +from gitdb import OStream from gitdb.utils.compat import xrange import sys diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 3563fcfbe..5b5c40e0d 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -4,9 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os -from gitdb.test.lib import * -import shutil -import tempfile +from gitdb.test.lib import TestBase #{ Invvariants diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 63856e218..b18e31ae6 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -9,11 +9,11 @@ from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB +from gitdb.utils.compat import xrange import sys import os from time import time -import random from nose import SkipTest diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index c66e60cba..297426303 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -38,7 +38,6 @@ def test_pack_writing(self): ni = 5000 count = 0 - total_size = 0 st = time() for sha in pdb.sha_iter(): count += 1 diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 010003d4b..6c8f71520 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -4,13 +4,13 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" from lib import TestBigRepoR -from gitdb.db import * -from gitdb.base import * -from gitdb.stream import * +from gitdb.db import LooseObjectDB +from gitdb.stream import IStream + from gitdb.util import ( - pool, - bin_to_hex - ) + pool, + bin_to_hex +) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size @@ -19,19 +19,15 @@ ChannelThreadTask, ) -from cStringIO import StringIO from time import time import os import sys -import stat -import subprocess from lib import ( - TestBigRepoR, make_memory_file, with_rw_directory - ) +) #{ Utilities diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 4cca7dabe..578c29f73 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -9,7 +9,15 @@ DeriveTest, ) -from gitdb import * +from gitdb import ( + OInfo, + OPackInfo, + ODeltaPackInfo, + OStream, + OPackStream, + ODeltaPackStream, + IStream +) from gitdb.util import ( NULL_BIN_SHA ) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index c644b8849..433518c25 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,7 +3,10 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from gitdb.test.lib import * +from gitdb.test.lib import ( + TestBase, + fixture_path +) from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 6b67ad887..3ab2fec07 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -6,7 +6,6 @@ from gitdb.test.lib import ( TestBase, with_rw_directory, - with_packs_rw, fixture_path ) @@ -27,7 +26,6 @@ from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha from gitdb.utils.compat import xrange -from itertools import chain try: from itertools import izip @@ -37,7 +35,6 @@ from nose import SkipTest import os -import sys import tempfile diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 984d6e1d7..671a146da 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -7,17 +7,18 @@ from gitdb.test.lib import ( TestBase, DummyStream, - Sha1Writer, make_bytes, make_object, fixture_path ) -from gitdb import * -from gitdb.util import ( - NULL_HEX_SHA, - hex_to_bin +from gitdb import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + LooseObjectDB, + Sha1Writer ) +from gitdb.util import hex_to_bin import zlib from gitdb.typ import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index ec9a86ce7..e79355aaf 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -5,7 +5,6 @@ """Test for object db""" import tempfile import os -import sys from gitdb.test.lib import TestBase from gitdb.util import ( diff --git a/gitdb/util.py b/gitdb/util.py index 77a0023e6..8843e8c88 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -77,7 +77,10 @@ def unpack_from(fmt, data, offset=0): fsync = os.fsync # Backwards compatibility imports -from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA +from gitdb.const import ( + NULL_BIN_SHA, + NULL_HEX_SHA +) #} END Aliases @@ -124,6 +127,7 @@ def make_sha(source=''.encode("ascii")): try: return hashlib.sha1(source) except NameError: + import sha sha1 = sha.sha(source) return sha1 diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index eec319f3d..a2640fd23 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -6,8 +6,10 @@ from itertools import izip xrange = xrange except ImportError: + # py3 izip = zip xrange = range +# end handle python version try: # Python 2 @@ -15,11 +17,15 @@ memoryview = buffer except NameError: # Python 3 has no `buffer`; only `memoryview` + # However, it's faster to just slice the object directly, maybe it keeps a view internally def buffer(obj, offset, size=None): if size is None: - return memoryview(obj)[offset:] + # return memoryview(obj)[offset:] + return obj[offset:] else: - return memoryview(obj[offset:offset+size]) + # return memoryview(obj)[offset:offset+size] + return obj[offset:offset+size] + # end buffer reimplementation memoryview = memoryview From bf942a913d69eb2079f9e82888aaccf2f6222643 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 13:31:32 +0100 Subject: [PATCH 0262/1790] Fully removed all async dependencies --- .gitmodules | 3 - doc/source/changes.rst | 7 +++ gitdb/__init__.py | 4 +- gitdb/db/base.py | 55 ---------------- gitdb/db/mem.py | 8 +-- gitdb/db/pack.py | 4 -- gitdb/ext/async | 1 - gitdb/test/__init__.py | 12 ---- gitdb/test/db/lib.py | 83 ------------------------ gitdb/test/db/test_git.py | 1 - gitdb/test/db/test_loose.py | 1 - gitdb/test/performance/test_stream.py | 90 +-------------------------- gitdb/test/test_example.py | 23 ------- gitdb/util.py | 10 --- setup.py | 6 +- 15 files changed, 14 insertions(+), 294 deletions(-) delete mode 160000 gitdb/ext/async diff --git a/.gitmodules b/.gitmodules index 978105388..d85b15c9f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "async"] - path = gitdb/ext/async - url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 839bf16a8..f544f76c0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +0.6.0 +***** + +* Added support got python 3.X +* Removed all `async` dependencies and all `*_async` versions of methods with it. + ***** 0.5.4 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 72b5ab085..165993fc9 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -10,7 +10,7 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - for module in ('async', 'smmap'): + for module in ('smmap',): sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) try: @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 5) +version_info = (0, 6, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index c534705b7..eac54ec38 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,7 +4,6 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, join, LazyMixin, hex_to_bin @@ -15,10 +14,6 @@ AmbiguousObjectName ) -from async import ( - ChannelThreadTask -) - from itertools import chain from functools import reduce @@ -41,47 +36,18 @@ def has_object(self, sha): binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") - def has_object_async(self, reader): - """Return a reader yielding information about the membership of objects - as identified by shas - :param reader: Reader yielding 20 byte shas. - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate - whether the given sha exists in the database or not""" - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - def info(self, sha): """ :return: OInfo instance :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def info_async(self, reader): - """Retrieve information of a multitude of objects asynchronously - :param reader: Channel yielding the sha's of the objects of interest - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" - task = ChannelThreadTask(reader, str(self.info_async), self.info) - return pool.add_task(task) - def stream(self, sha): """:return: OStream instance :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def stream_async(self, reader): - """Retrieve the OStream of multiple objects - :param reader: see ``info`` - :param max_threads: see ``ObjectDBW.store`` - :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to - read all OStreams at once. Instead, read them individually using reader.read(x) - where x is small enough.""" - # base implementation just uses the stream method repeatedly - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) - return pool.add_task(task) - def size(self): """:return: amount of objects in this database""" raise NotImplementedError() @@ -129,27 +95,6 @@ def store(self, istream): :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - def store_async(self, reader): - """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as - they are computed. - - :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, - or its error attribute will be set to the exception informing about the error. - - :param reader: async.Reader yielding IStream instances. - The same instances will be used in the output channel as were received - in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how - many items have actually been produced.""" - # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) - return pool.add_task(task) - #} END edit interface diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index a22454685..1aa0d511f 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -32,10 +32,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): """A memory database stores everything to memory, providing fast IO and object retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already - exists in the target storage before introducing actual IO - - **Note:** memory is currently not threadsafe, hence the async methods cannot be used - for storing""" + exists in the target storage before introducing actual IO""" def __init__(self): super(MemoryDB, self).__init__() @@ -62,9 +59,6 @@ def store(self, istream): return istream - def store_async(self, reader): - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - def has_object(self, sha): return sha in self._cache diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 4d0a7f8b3..eaf431a22 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -125,10 +125,6 @@ def store(self, istream): inefficient""" raise UnsupportedOperation() - def store_async(self, reader): - # TODO: add ObjectDBRW before implementing this - raise NotImplementedError() - #} END object db write diff --git a/gitdb/ext/async b/gitdb/ext/async deleted file mode 160000 index b930ee15c..000000000 --- a/gitdb/ext/async +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index ca104c0c5..8a681e428 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -2,15 +2,3 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php - -import gitdb.util - -#{ Initialization -def _init_pool(): - """Assure the pool is actually threaded""" - size = 2 - print("Setting ThreadPool to %i" % size) - gitdb.util.pool.set_size(size) - - -#} END initialization diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 67958c366..962d4bc2a 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -10,8 +10,6 @@ TestBase ) - - from gitdb.stream import ( Sha1Writer, ZippedStoreShaWriter @@ -28,8 +26,6 @@ from gitdb.utils.encoding import force_bytes from gitdb.utils.compat import xrange -from async import IteratorReader - from io import BytesIO from struct import pack @@ -132,82 +128,3 @@ def _assert_object_writing(self, db): # END for each data set # END for each dry_run mode - def _assert_object_writing_async(self, db): - """Test generic object writing using asynchronous access""" - ni = 5000 - def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - data = bytes(data_src + offset) - yield IStream(str_blob_type, len(data), BytesIO(data)) - # END for each item - # END generator utility - - # for now, we are very trusty here as we expect it to work if it worked - # in the single-stream case - - # write objects - reader = IteratorReader(istream_generator()) - istream_reader = db.store_async(reader) - istreams = istream_reader.read() # read all - assert istream_reader.task().error() is None - assert len(istreams) == ni - - for stream in istreams: - assert stream.error is None - assert len(stream.binsha) == 20 - assert isinstance(stream, IStream) - # END assert each stream - - # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.binsha for istream in istreams ) - hasobject_reader = db.has_object_async(reader) - count = 0 - for sha, has_object in hasobject_reader: - assert has_object - count += 1 - # END for each sha - assert count == ni - - # read the objects we have just written - reader = IteratorReader( istream.binsha for istream in istreams ) - ostream_reader = db.stream_async(reader) - - # read items individually to prevent hitting possible sys-limits - count = 0 - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert ostream_reader.task().error() is None - assert count == ni - - # get info about our items - reader = IteratorReader( istream.binsha for istream in istreams ) - info_reader = db.info_async(reader) - - count = 0 - for oinfo in info_reader: - assert isinstance(oinfo, OInfo) - count += 1 - # END for each oinfo instance - assert count == ni - - - # combined read-write using a converter - # add 2500 items, and obtain their output streams - nni = 2500 - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - - istream_reader = db.store_async(reader) - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = db.stream_async(istream_reader) - - count = 0 - # read it individually, otherwise we might run into the ulimit - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert count == nni diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 4bce7dac8..56899e579 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -48,4 +48,3 @@ def test_writing(self, path): # its possible to write objects self._assert_object_writing(gdb) - self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 1299f7b86..1d6af9c99 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -18,7 +18,6 @@ def test_basics(self, path): # write data self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) # verify sha iteration and size shas = list(ldb.sha_iter()) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 6c8f71520..929c7e537 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -7,18 +7,9 @@ from gitdb.db import LooseObjectDB from gitdb.stream import IStream -from gitdb.util import ( - pool, - bin_to_hex -) -from gitdb.typ import str_blob_type +from gitdb.util import bin_to_hex from gitdb.fun import chunk_size -from async import ( - IteratorReader, - ChannelThreadTask, - ) - from time import time import os import sys @@ -43,15 +34,6 @@ def read_chunked_stream(stream): return stream -class TestStreamReader(ChannelThreadTask): - """Expects input streams and reads them in chunks. It will read one at a time, - requireing a queue chunk of size 1""" - def __init__(self, *args): - super(TestStreamReader, self).__init__(*args) - self.fun = read_chunked_stream - self.max_chunksize = 1 - - #} END utilities class TestObjDBPerformance(TestBigRepoR): @@ -119,73 +101,3 @@ def test_large_data_streaming(self, path): # del db file so we keep something to do os.remove(db_file) # END for each randomization factor - - - # multi-threaded mode - # want two, should be supported by most of todays cpus - pool.set_size(2) - total_kib = 0 - nsios = len(string_ios) - for stream in string_ios: - stream.seek(0) - total_kib += len(stream.getvalue()) / 1000 - # END rewind - - def istream_iter(): - for stream in string_ios: - stream.seek(0) - yield IStream(str_blob_type, len(stream.getvalue()), stream) - # END for each stream - # END util - - # write multiple objects at once, involving concurrent compression - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - st = time() - istreams = istream_reader.read(nsios) - assert len(istreams) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # decompress multiple at once, by reading them - # chunk size is not important as the stream will not really be decompressed - - # until its read - istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.task().max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # store the files, and read them back. For the reading, we use a task - # as well which is chunked into one item per task. Reading all will - # very quickly result in two threads handling two bytestreams of - # chained compression/decompression streams - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - istream_to_sha = lambda items: [ i.binsha for i in items ] - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 433518c25..aa43a093f 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -9,12 +9,9 @@ ) from gitdb import IStream from gitdb.db import LooseObjectDB -from gitdb.util import pool from io import BytesIO -from async import IteratorReader - class TestExamples(TestBase): def test_base(self): @@ -45,23 +42,3 @@ def test_base(self): # now the sha is set assert len(istream.binsha) == 20 assert ldb.has_object(istream.binsha) - - - # async operation - # Create a reader from an iterator - reader = IteratorReader(ldb.sha_iter()) - - # get reader for object streams - info_reader = ldb.stream_async(reader) - - # read one - info = info_reader.read(1)[0] - - # read all the rest until depletion - ostreams = info_reader.read() - - # set the pool to use two threads - pool.set_size(2) - - # synchronize the mode of operation - pool.set_size(0) diff --git a/gitdb/util.py b/gitdb/util.py index 8843e8c88..93ba7f0ec 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -10,7 +10,6 @@ from io import StringIO -from async import ThreadPool from smmap import ( StaticWindowMapManager, SlidingWindowMapManager, @@ -43,15 +42,6 @@ def unpack_from(fmt, data, offset=0): # END own unpack_from implementation -#{ Globals - -# A pool distributing tasks, initially with zero threads, hence everything -# will be handled in the main thread -pool = ThreadPool(0) - -#} END globals - - #{ Aliases hex_to_bin = binascii.a2b_hex diff --git a/setup.py b/setup.py index 639370471..63ec5ddb3 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 5) +version_info = (0, 6, 0) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, @@ -88,8 +88,8 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.2)', 'smmap (>=0.8.3)'), - install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), + requires=('smmap (>=0.8.3)'), + install_requires=('smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 641b64c9f48139cf06774805d32892012fb9b82d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 18:31:17 +0100 Subject: [PATCH 0263/1790] Now tests work consistently in py2 and 3 It's a nice way of saying that there is still one failing, consistently. --- gitdb/const.py | 5 +- gitdb/db/base.py | 7 +- gitdb/db/loose.py | 2 +- gitdb/fun.py | 229 +++++++++++++++++++++++++------------ gitdb/pack.py | 21 ++-- gitdb/stream.py | 8 +- gitdb/test/db/test_pack.py | 2 +- gitdb/typ.py | 10 +- gitdb/utils/encoding.py | 4 +- 9 files changed, 178 insertions(+), 110 deletions(-) diff --git a/gitdb/const.py b/gitdb/const.py index 147f79cb9..6391d796f 100644 --- a/gitdb/const.py +++ b/gitdb/const.py @@ -1,5 +1,4 @@ -from gitdb.utils.encoding import force_bytes - -NULL_BYTE = force_bytes("\0") +BYTE_SPACE = b' ' +NULL_BYTE = b'\0' NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index eac54ec38..a670eea63 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -9,6 +9,7 @@ hex_to_bin ) +from gitdb.utils.encoding import force_text from gitdb.exc import ( BadObject, AmbiguousObjectName @@ -122,8 +123,6 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" - from gitdb.utils.encoding import force_text - return join(self._root_path, force_text(rela_path)) #} END interface @@ -234,12 +233,12 @@ def update_cache(self, force=False): def partial_to_complete_sha_hex(self, partial_hexsha): """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str) :param partial_hexsha: hexsha with less than 40 byte :raise AmbiguousObjectName: """ databases = list() _databases_recursive(self, databases) - + partial_hexsha = force_text(partial_hexsha) len_partial_hexsha = len(partial_hexsha) if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 3abdaa96f..374302611 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -109,7 +109,7 @@ def readable_db_object_path(self, hexsha): def partial_to_complete_sha_hex(self, partial_hexsha): """:return: 20 byte binary sha1 string which matches the given name uniquely - :param name: hexadecimal partial name + :param name: hexadecimal partial name (bytes or ascii string) :raise AmbiguousObjectName: :raise BadObject: """ candidate = None diff --git a/gitdb/fun.py b/gitdb/fun.py index 80f01e6b2..064680adb 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -14,7 +14,9 @@ from itertools import islice from functools import reduce -from gitdb.utils.compat import izip, buffer, xrange +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_text +from gitdb.utils.compat import izip, buffer, xrange, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -30,12 +32,12 @@ delta_types = (OFS_DELTA, REF_DELTA) type_id_to_type_map = { - 0 : "", # EXT 1 + 0 : b'', # EXT 1 1 : str_commit_type, 2 : str_tree_type, 3 : str_blob_type, 4 : str_tag_type, - 5 : "", # EXT 2 + 5 : b'', # EXT 2 OFS_DELTA : "OFS_DELTA", # OFFSET DELTA REF_DELTA : "REF_DELTA" # REFERENCE DELTA } @@ -394,11 +396,9 @@ def loose_object_header_info(m): :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" - from gitdb.const import NULL_BYTE - decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE) return type_name, int(size) @@ -413,12 +413,21 @@ def pack_object_header_info(data): type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size - while c & 0x80: - c = byte_ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop + if PY3: + while c & 0x80: + c = data[i] + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + else: + while c & 0x80: + c = ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + # end performance at expense of maintenance ... return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): @@ -429,16 +438,29 @@ def create_pack_object_header(obj_type, obj_size): :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte - hdr = str() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - #END until size is consumed - hdr += chr(c) + if PY3: + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr.append(c) + else: + hdr = bytes() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + # end handle interpreter return hdr def msb_size(data, offset=0): @@ -449,24 +471,36 @@ def msb_size(data, offset=0): i = 0 l = len(data) hit_msb = False - while i < l: - c = byte_ord(data[i+offset]) - size |= (c & 0x7f) << i*7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range + if PY3: + while i < l: + c = data[i+offset] + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + else: + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") return i+offset, size def loose_object_header(type, size): """ - :return: string representing the loose object header, which is immediately + :return: bytes representing the loose object header, which is immediately followed by the content stream of size 'size'""" - return "%s %i\0" % (type, size) + return ('%s %i\0' % (force_text(type), size)).encode('ascii') def write_object(type, size, read, write, chunk_size=chunk_size): """ @@ -611,48 +645,93 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i+c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data + if PY3: + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + else: + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + # end save byte_ord call and prevent performance regression in py2 # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" diff --git a/gitdb/pack.py b/gitdb/pack.py index 87818b6e7..375cc59a9 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -86,8 +86,6 @@ def pack_object_at(cursor, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - from gitdb.utils.encoding import force_bytes - data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -96,11 +94,11 @@ def pack_object_at(cursor, offset, as_stream): # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 delta_offset = c & 0x7f while c & 0x80: - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 delta_offset += 1 delta_offset = (delta_offset << 7) + (c & 0x7f) @@ -118,12 +116,10 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - buff = buffer(data, total_rela_offset) - stream = DecompressMemMapReader(buff, False, uncomp_size) + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - delta_info = force_bytes(delta_info) return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: @@ -204,7 +200,7 @@ def write(self, pack_sha, write): # fanout tmplist = list((0,)*256) # fanout or list with 64 bit offsets for t in self._objs: - tmplist[ord(t[0][0])] += 1 + tmplist[byte_ord(t[0][0])] += 1 #END prepare fanout for i in xrange(255): v = tmplist[i] @@ -215,7 +211,7 @@ def write(self, pack_sha, write): # sha1 ordered # save calls, that is push them into c - sha_write(''.join(t[0] for t in self._objs)) + sha_write(b''.join(t[0] for t in self._objs)) # crc32 for t in self._objs: @@ -258,7 +254,7 @@ class PackIndexFile(LazyMixin): # used in v2 indices _sha_list_offset = 8 + 1024 - index_v2_signature = '\377tOc' + index_v2_signature = b'\xfftOc' index_version_default = 2 def __init__(self, indexpath): @@ -446,13 +442,14 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this index file - :param partial_bin_sha: an at least two bytes of a partial binary sha + :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes :param canonical_length: lenght of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") + assert isinstance(partial_bin_sha, bytes), "partial_bin_sha must be bytes" first_byte = byte_ord(partial_bin_sha[0]) get_sha = self.sha @@ -680,7 +677,7 @@ def _set_cache_(self, attr): else: iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) - iter_offsets_plus_one.next() + next(iter_offsets_plus_one) consecutive = izip(iter_offsets, iter_offsets_plus_one) offset_map = dict(consecutive) diff --git a/gitdb/stream.py b/gitdb/stream.py index 5daf01cb6..43aa8e368 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -25,7 +25,7 @@ close, ) -from gitdb.const import NULL_BYTE +from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.compat import buffer from gitdb.utils.encoding import force_bytes @@ -102,7 +102,7 @@ def _parse_header_info(self): self._s = maxb hdr = self.read(maxb) hdrend = hdr.find(NULL_BYTE) - typ, size = hdr[:hdrend].split(" ".encode("ascii")) + typ, size = hdr[:hdrend].split(BYTE_SPACE) size = int(size) self._s = size @@ -378,8 +378,6 @@ def _set_cache_too_slow_without_c(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" - from gitdb.utils.compat import buffer - # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the # delta is not peaked into, causing less overhead. @@ -421,7 +419,7 @@ def _set_cache_brute_(self, attr): # For the actual copying, we use a seek and write pattern of buffer # slices. final_target_size = None - for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): + for (dbuf, offset, src_size, target_size), dstream in zip(reversed(buffer_info_list), reversed(self._dstreams)): # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 1177cf962..963a71af7 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -73,4 +73,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/typ.py b/gitdb/typ.py index edd1f2731..bc7ba5828 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,9 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -from gitdb.utils.encoding import force_bytes - -str_blob_type = force_bytes("blob") -str_commit_type = force_bytes("commit") -str_tree_type = force_bytes("tree") -str_tag_type = force_bytes("tag") +str_blob_type = b'blob' +str_commit_type = b'commit' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 12164e756..617b51c83 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -11,9 +11,6 @@ def force_bytes(data, encoding="utf-8"): if isinstance(data, bytes): return data - if isinstance(data, compat.memoryview): - return bytes(data) - if isinstance(data, string_types): return data.encode(encoding) @@ -27,6 +24,7 @@ def force_text(data, encoding="utf-8"): return data.decode(encoding) if not isinstance(data, bytes): + assert False, "Shouldn't be here" data = force_bytes(data, encoding) if compat.PY3: From 8ae4e9579a263684c6b760aec2869be480ff22ba Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 18:46:49 +0100 Subject: [PATCH 0264/1790] reduced usage of force_bytes as clients are expected to pass bytes. It was useful for debugging though, maybe an explicit type assertions would help others ? As 'others' will be gitpython, I suppose I can handle it myself --- gitdb/stream.py | 6 +----- gitdb/test/db/lib.py | 7 +++---- gitdb/test/db/test_ref.py | 8 +++++--- gitdb/utils/encoding.py | 6 +----- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 43aa8e368..e32fcf380 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -543,9 +543,9 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written + :param data: byte object :return: length of incoming data""" - data = force_bytes(data) self.sha1.update(data) return len(data) @@ -590,8 +590,6 @@ def __getattr__(self, attr): return getattr(self.buf, attr) def write(self, data): - data = force_bytes(data) - alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) @@ -634,8 +632,6 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written :return: lenght of incoming data""" - data = force_bytes(data) - self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 962d4bc2a..af6d9e0fd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,6 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type -from gitdb.utils.encoding import force_bytes from gitdb.utils.compat import xrange from io import BytesIO @@ -37,7 +36,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world" + two_lines = b'1234\nhello world' all_data = (two_lines, ) def _assert_object_writing_simple(self, db): @@ -83,7 +82,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii"))) + istream = IStream(str_blob_type, len(data), BytesIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -99,7 +98,7 @@ def _assert_object_writing(self, db): assert info.size == len(data) ostream = db.stream(sha) - assert ostream.read() == force_bytes(data) + assert ostream.read() == data assert ostream.type == str_blob_type assert ostream.size == len(data) else: diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index a1387ee26..db930827b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -2,7 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + fixture_path +) from gitdb.db import ReferenceDB from gitdb.util import ( @@ -24,8 +28,6 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - NULL_BIN_SHA = '\0'.encode("ascii") * 20 - alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) assert len(rdb.databases()) == 0 diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 617b51c83..2d03ad30c 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -7,7 +7,7 @@ string_types = (basestring, ) text_type = unicode -def force_bytes(data, encoding="utf-8"): +def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -23,10 +23,6 @@ def force_text(data, encoding="utf-8"): if isinstance(data, string_types): return data.decode(encoding) - if not isinstance(data, bytes): - assert False, "Shouldn't be here" - data = force_bytes(data, encoding) - if compat.PY3: return text_type(data, encoding) else: From 7fd369c8549a975efd74c312aa91194b4569b99e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 19:49:40 +0100 Subject: [PATCH 0265/1790] setup.py works now, and binary python module can now be loaded as well. --- gitdb/_delta_apply.c | 2 -- gitdb/fun.py | 1 - setup.py | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index f03e7ea6d..8b0f8e064 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -506,7 +506,6 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded // and added to the spot marked by sdc -inline uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) { uint num_bytes = 0; @@ -559,7 +558,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) // destination memory. The individual chunks written will be a byte copy of the source // data chunk stream // Return: number of chunks in the slice -inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) { assert(DIV_lbound(src) <= tofs); diff --git a/gitdb/fun.py b/gitdb/fun.py index 064680adb..b7662b495 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -758,7 +758,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - # NOQA from _perf import connect_deltas except ImportError: pass diff --git a/setup.py b/setup.py index 63ec5ddb3..e01c6b475 100755 --- a/setup.py +++ b/setup.py @@ -83,12 +83,12 @@ def get_data_files(self): author = __author__, author_email = __contact__, url = __homepage__, - packages = ('gitdb', 'gitdb.db'), + packages = ('gitdb', 'gitdb.db', 'gitdb.utils'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('smmap (>=0.8.3)'), + requires=('smmap (>=0.8.3)', ), install_requires=('smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From f26c869025bc31d3920726a63b15fdb420b1d215 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 09:43:19 +0100 Subject: [PATCH 0266/1790] Performance tests are now part of the test-suite. By default, a small repository will be tested, which doesn't take that long actually (~20s) Additionally, that way we enforce correctness tests, which didn't run by default previously. As we are handling data here, we must be sure that it's handled correctly, thus the tests should run. --- gitdb/test/lib.py | 4 +-- gitdb/test/performance/lib.py | 27 ++++++++----------- gitdb/test/performance/test_pack.py | 21 +++++++-------- gitdb/test/performance/test_pack_streaming.py | 17 ++++++------ gitdb/test/performance/test_stream.py | 20 +++++++------- 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index d88ec8b43..ba653c91f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -10,7 +10,7 @@ import random from array import array -from io import StringIO +from io import BytesIO import glob import unittest @@ -120,7 +120,7 @@ def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) + return len(d), BytesIO(d) #} END routines diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 5b5c40e0d..ec45cf3a7 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -4,6 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os +import logging from gitdb.test.lib import TestBase @@ -12,17 +13,6 @@ #} END invariants -#{ Utilities -def resolve_or_fail(env_var): - """:return: resolved environment variable or raise EnvironmentError""" - try: - return os.environ[env_var] - except KeyError: - raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) - # END exception handling - -#} END utilities - #{ Base Classes @@ -39,14 +29,19 @@ class TestBigRepoR(TestBase): head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' #} END invariants - @classmethod - def setUpAll(cls): + def setUp(self): try: - super(TestBigRepoR, cls).setUpAll() + super(TestBigRepoR, self).setUp() except AttributeError: pass - cls.gitrepopath = resolve_or_fail(k_env_git_repo) - assert cls.gitrepopath.endswith('.git') + + self.gitrepopath = os.environ.get(k_env_git_repo) + if not self.gitrepopath: + logging.info("You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository") + ospd = os.path.dirname + self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') + # end assure gitrepo is set + assert self.gitrepopath.endswith('.git') #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b18e31ae6..b52e46fec 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" -from lib import ( +from __future__ import print_function + +from gitdb.test.performance.lib import ( TestBigRepoR - ) +) from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB @@ -15,8 +17,6 @@ import os from time import time -from nose import SkipTest - class TestPackedDBPerformance(TestBigRepoR): def test_pack_random_access(self): @@ -27,7 +27,7 @@ def test_pack_random_access(self): sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) - print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info @@ -41,7 +41,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) # END for each random mode # query info and streams only @@ -51,7 +51,7 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) # END for each function # retrieve stream and read all @@ -65,13 +65,12 @@ def test_pack_random_access(self): total_size += stream.size elapsed = time() - st total_kib = total_size / 1000 - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) def test_correctness(self): - raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time - print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" + print("Endurance run: verify streaming of objects (crc and sha)", file=sys.stderr) for crc in range(2): count = 0 st = time() @@ -88,6 +87,6 @@ def test_correctness(self): # END for each index # END for each entity elapsed = time() - st - print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed), file=sys.stderr) # END for each verify mode diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 297426303..b1001f85b 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" -from lib import ( +from __future__ import print_function + +from gitdb.test.performance.lib import ( TestBigRepoR - ) +) from gitdb.db.pack import PackedDB from gitdb.stream import NullStream @@ -14,7 +16,6 @@ import os import sys from time import time -from nose import SkipTest class CountedNullStream(NullStream): __slots__ = '_bw' @@ -36,7 +37,7 @@ def test_pack_writing(self): ostream = CountedNullStream() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - ni = 5000 + ni = 1000 count = 0 st = time() for sha in pdb.sha_iter(): @@ -46,17 +47,17 @@ def test_pack_writing(self): break #END gather objects for pack-writing elapsed = time() - st - print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed), file=sys.stderr) st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 - print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) def test_stream_reading(self): - raise SkipTest() + # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # streaming only, meant for --with-profile runs @@ -74,5 +75,5 @@ def test_stream_reading(self): count += 1 elapsed = time() - st total_kib = total_size / 1000 - print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 929c7e537..9d695a047 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" -from lib import TestBigRepoR +from __future__ import print_function + +from gitdb.test.performance.lib import TestBigRepoR from gitdb.db import LooseObjectDB -from gitdb.stream import IStream +from gitdb import IStream from gitdb.util import bin_to_hex from gitdb.fun import chunk_size @@ -15,7 +17,7 @@ import sys -from lib import ( +from gitdb.test.lib import ( make_memory_file, with_rw_directory ) @@ -49,11 +51,11 @@ def test_large_data_streaming(self, path): # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' - print >> sys.stderr, "Creating %s data ..." % desc + print("Creating %s data ..." % desc, file=sys.stderr) st = time() size, stream = make_memory_file(self.large_data_size_bytes, randomize) elapsed = time() - st - print >> sys.stderr, "Done (in %f s)" % elapsed + print("Done (in %f s)" % elapsed, file=sys.stderr) string_ios.append(stream) # writing - due to the compression it will seem faster than it is @@ -66,7 +68,7 @@ def test_large_data_streaming(self, path): size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) # reading all at once st = time() @@ -76,7 +78,7 @@ def test_large_data_streaming(self, path): stream.seek(0) assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) # reading in chunks of 1 MiB @@ -93,10 +95,10 @@ def test_large_data_streaming(self, path): elapsed_readchunks = time() - st stream.seek(0) - assert ''.join(chunks) == stream.getvalue() + assert b''.join(chunks) == stream.getvalue() cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) # del db file so we keep something to do os.remove(db_file) From 232cf20efaef4218a71c348b0f59c5e59c59cbdb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 10:43:28 +0100 Subject: [PATCH 0267/1790] Fixed incorrect computation of compressed bytes read in zlib decompression stream. --- gitdb/stream.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index e32fcf380..0332df6e4 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -275,7 +275,10 @@ def read(self, size=-1): # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data # if we hit the end of the stream - self._cbr += len(indata) - len(self._zip.unconsumed_tail) + # NOTE: For some reason, the code worked for a long time with substracting unconsumed_tail + # Now, however, it really asks for unused_data, and I wonder whether unconsumed_tail still has to + # be substracted. On the plus side, the tests work, so it seems to be ok for py 2.7 and 3.4 + self._cbr += len(indata) - len(self._zip.unconsumed_tail) - len(self._zip.unused_data) self._br += len(dcompdat) if dat: From 6f71b8a90250ed03f2e7de2e61d22e84c0fbb2ff Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:04:23 +0100 Subject: [PATCH 0268/1790] Fixed .travis file to allow tests to work correctly. Previously, submodules were not initalized, which could have had an effect ... . Even though it shouldn't, but lets just try it. --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf1d13666..db7c2bc66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,11 @@ python: - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) -script: nosetests +install: + - git submodule update --init --recursive + - pip install coveralls +script: + - nosetests +after_success: + - coveralls + From b9d189d35073cc80ddbfa61269c65785264880f3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:29:18 +0100 Subject: [PATCH 0269/1790] Added requirements.txt for pip, and optimized test-suite performance on travis. With a bit of luck, this one will just work now. --- .travis.yml | 1 - README.rst | 33 +++++++++++-------- doc/source/intro.rst | 8 ++--- gitdb/test/db/test_git.py | 4 ++- gitdb/test/lib.py | 16 +++++++++ gitdb/test/performance/__init__.py | 1 + gitdb/test/performance/test_pack.py | 5 ++- gitdb/test/performance/test_pack_streaming.py | 3 ++ gitdb/test/performance/test_stream.py | 6 ++-- requirements.txt | 2 ++ setup.py | 2 +- 11 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 gitdb/test/performance/__init__.py create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index db7c2bc66..10b5e161b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) install: - - git submodule update --init --recursive - pip install coveralls script: - nosetests diff --git a/README.rst b/README.rst index f97d5c31e..194e24658 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,7 @@ GitDB ===== -GitDB allows you to access bare git repositories for reading and writing. It -aims at allowing full access to loose objects as well as packs with performance -and scalability in mind. It operates exclusively on streams, allowing to operate -on large objects with a small memory footprint. +GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. Installation ============ @@ -23,13 +20,13 @@ From `PyPI `_ REQUIREMENTS ============ -* Python Nose - for running the tests +* Python Nose - for running the tests SOURCE ====== The source is available in a git repository at gitorious and github: -git://github.com/gitpython-developers/gitdb.git +https://github.com/gitpython-developers/gitdb Once the clone is complete, please be sure to initialize the submodules using @@ -40,17 +37,25 @@ Run the tests with nosetests -MAILING LIST -============ -http://groups.google.com/group/git-python - -ISSUE TRACKER -============= +DEVELOPMENT +=========== .. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - -https://github.com/gitpython-developers/gitdb/issues + +.. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png + :target: https://coveralls.io/r/gitpython-developers/gitdb + +The library is considered mature, and not under active development. It's primary (known) use is in git-python. + +INFRASTRUCTURE +============== + +* Mailing List + * http://groups.google.com/group/git-python + +* Issue Tracker + * https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8fc0ec098..434138616 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -11,9 +11,9 @@ Interfaces are used to describe the API, making it easy to provide alternate imp ================ Installing GitDB ================ -Its easiest to install gitdb using the *easy_install* program, which is part of the `setuptools`_:: +Its easiest to install gitdb using the *pip* program:: - $ easy_install gitdb + $ pip install gitdb As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. @@ -31,10 +31,8 @@ Source Repository ================= The latest source can be cloned using git from github: - * git://github.com/gitpython-developers/gitdb.git + * https://github.com/gitpython-developers/gitdb License Information =================== *GitDB* is licensed under the New BSD License. - -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 56899e579..e141c2ba0 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -24,9 +24,11 @@ def test_reading(self): gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) - assert gdb.size() > 200 + ni = 50 + assert gdb.size() >= ni sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() + sha_list = sha_list[:ni] # speed up tests ... # This is actually a test for compound functionality, but it doesn't diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ba653c91f..d09b1cb8e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -18,6 +18,7 @@ import shutil import os import gc +from functools import wraps #{ Bases @@ -30,6 +31,21 @@ class TestBase(unittest.TestCase): #{ Decorators +def skip_on_travis_ci(func): + """All tests decorated with this one will raise SkipTest when run on travis ci. + Use it to workaround difficult to solve issues + NOTE: copied from bcore (https://github.com/Byron/bcore)""" + @wraps(func) + def wrapper(self, *args, **kwargs): + if 'TRAVIS' in os.environ: + import nose + raise nose.SkipTest("Cannot run on travis-ci") + # end check for travis ci + return func(self, *args, **kwargs) + # end wrapper + return wrapper + + def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" diff --git a/gitdb/test/performance/__init__.py b/gitdb/test/performance/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/gitdb/test/performance/__init__.py @@ -0,0 +1 @@ + diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b52e46fec..db3b48de5 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -12,13 +12,15 @@ from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB from gitdb.utils.compat import xrange +from gitdb.test.lib import skip_on_travis_ci import sys import os from time import time class TestPackedDBPerformance(TestBigRepoR): - + + @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -67,6 +69,7 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index b1001f85b..fe160ea54 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -12,6 +12,7 @@ from gitdb.db.pack import PackedDB from gitdb.stream import NullStream from gitdb.pack import PackEntity +from gitdb.test.lib import skip_on_travis_ci import os import sys @@ -31,6 +32,7 @@ def write(self, d): class TestPackStreamingPerformance(TestBigRepoR): + @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well @@ -56,6 +58,7 @@ def test_pack_writing(self): print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) + @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 9d695a047..84c9dea3f 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -19,7 +19,8 @@ from gitdb.test.lib import ( make_memory_file, - with_rw_directory + with_rw_directory, + skip_on_travis_ci ) @@ -42,7 +43,8 @@ class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*50 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - + + @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..8a4cd3979 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +gitdb +smmap>=0.8.3 \ No newline at end of file diff --git a/setup.py b/setup.py index e01c6b475..dc142c518 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def get_data_files(self): license = "BSD License", zip_safe=False, requires=('smmap (>=0.8.3)', ), - install_requires=('smmap >= 0.8.0'), + install_requires=('smmap >= 0.8.3'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 0bb576427f899631fbbbb822c6d3058174f96847 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:37:12 +0100 Subject: [PATCH 0270/1790] Allow our clone to be deeper to help tests to work --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 10b5e161b..8a9e0afa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ python: - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) +git: + # a higher depth is needed for one of the tests - lets fet + depth: 1000 install: - pip install coveralls script: From e7fdd949d0cb2c42c9217e3c7009eb28c6b53446 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 12:07:13 +0100 Subject: [PATCH 0271/1790] Now I am skipping a problematic test on travis CI. Maybe I can find a py 2.6 interpreter somewhere to reproduce it. --- .travis.yml | 2 +- gitdb/test/test_stream.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a9e0afa6..761edc19b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ git: install: - pip install coveralls script: - - nosetests + - nosetests -v after_success: - coveralls diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 671a146da..f8d9f5dcd 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -27,7 +27,8 @@ import tempfile import os - +import sys +from nose import SkipTest class TestStream(TestBase): """Test stream classes""" @@ -70,10 +71,16 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle special type def test_decompress_reader(self): + cache = dict() for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: - cdata = make_bytes(ds, randomize=False) + if ds in cache: + cdata = cache[ds] + else: + cdata = make_bytes(ds, randomize=False) + cache[ds] = cdata + # end handle caching (maybe helps on py2.6 ?) # zdata = zipped actual data # cdata = original content data @@ -121,6 +128,9 @@ def test_sha_writer(self): assert writer.sha() != prev_sha def test_compressed_writer(self): + if sys.version_info[:2] < (2,7) and os.environ.get('TRAVIS'): + raise SkipTest("For some reason, this test STALLS on travis ci on py2.6, but works on my centos py2.6 interpreter") + # end special case ... for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) From 0dcec5a27b341ce58e5ab169f91aa25b2cafec0c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 12:39:09 +0100 Subject: [PATCH 0272/1790] It seems zlib works differently in py26, and thus requires special handling. This also explains why the tests suddenly stopped working - after all, the interpreter changed ... . --- gitdb/stream.py | 15 ++++++++++----- gitdb/test/test_stream.py | 13 +------------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 0332df6e4..edd6dd2b1 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -7,6 +7,7 @@ import mmap import os +import sys import zlib from gitdb.fun import ( @@ -30,6 +31,7 @@ from gitdb.utils.encoding import force_bytes has_perf_mod = False +PY26 = sys.version_info[:2] < (2, 7) try: from _perf import apply_delta as c_apply_delta has_perf_mod = True @@ -275,10 +277,14 @@ def read(self, size=-1): # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data # if we hit the end of the stream - # NOTE: For some reason, the code worked for a long time with substracting unconsumed_tail - # Now, however, it really asks for unused_data, and I wonder whether unconsumed_tail still has to - # be substracted. On the plus side, the tests work, so it seems to be ok for py 2.7 and 3.4 - self._cbr += len(indata) - len(self._zip.unconsumed_tail) - len(self._zip.unused_data) + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + if PY26: + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # end handle very special case ... + self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) if dat: @@ -505,7 +511,6 @@ def new(cls, stream_list): if stream_list[-1].type_id in delta_types: raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream - return cls(stream_list) #} END interface diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f8d9f5dcd..50db44b1d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -27,8 +27,6 @@ import tempfile import os -import sys -from nose import SkipTest class TestStream(TestBase): """Test stream classes""" @@ -71,16 +69,10 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle special type def test_decompress_reader(self): - cache = dict() for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: - if ds in cache: - cdata = cache[ds] - else: - cdata = make_bytes(ds, randomize=False) - cache[ds] = cdata - # end handle caching (maybe helps on py2.6 ?) + cdata = make_bytes(ds, randomize=False) # zdata = zipped actual data # cdata = original content data @@ -128,9 +120,6 @@ def test_sha_writer(self): assert writer.sha() != prev_sha def test_compressed_writer(self): - if sys.version_info[:2] < (2,7) and os.environ.get('TRAVIS'): - raise SkipTest("For some reason, this test STALLS on travis ci on py2.6, but works on my centos py2.6 interpreter") - # end special case ... for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) From 59589e0fddfc9f3ea318dd7b3c0ef5b4e1bb665c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 14:57:01 +0100 Subject: [PATCH 0273/1790] Added pypi badges [ skip ci ] --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 327d66372..8c9ae4251 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap +[![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) +[![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) + Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: ```bash From f4b6b2508cf164be237b2fdeaca01be7153efe8c Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 22:24:34 +0100 Subject: [PATCH 0274/1790] Add tox env for flake8 linter Most people know about pep8 which enforce coding style. pyflakes goes a step beyond by analyzing the code. flake8 is basically a wrapper around both pep8 and pyflakes and comes with some additional checks. I find it very useful since you only need to require one package to have a lot of code issues reported to you. This patch provides a 'flake8' tox environement to easily install and run the utility on the code base. One simply has to: tox -eflake8 The env has been added to the default list of environement to have flake8 run by default. Configuration tweaking is done in setup.cfg [flake8] section. The repository in its current state does not pass checks We can later easily ensure there is no regression by adjusting Travis configuration to run this env. More informations about flake8: https://pypi.python.org/pypi/flake8 --- setup.cfg | 3 +++ tox.ini | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2a9acf13d..83a51c4b9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_wheel] universal = 1 + +[flake8] +exclude = .tox,.venv,build,dist,doc diff --git a/tox.ini b/tox.ini index e0e196418..35b813753 100644 --- a/tox.ini +++ b/tox.ini @@ -4,10 +4,14 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27, py33, py34 +envlist = flake8, py26, py27, py33, py34 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} deps = nose nosexcover + +[testenv:flake8] +commands = flake8 {posargs} +deps = flake8 From 01dac15d2d2ad916083559747c3df497921cdec9 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 22:26:49 +0100 Subject: [PATCH 0275/1790] pep8 linting E201 whitespace after '(' E203 whitespace before ',' E221 multiple spaces before operator E225 missing whitespace around operator E227 missing whitespace around bitwise or shift operator E231 missing whitespace after ',' E251 unexpected spaces around keyword / parameter equals W291 trailing whitespace W293 blank line contains whitespace E302 expected 2 blank lines, found 1 E303 too many blank lines (3) W391 blank line at end of file --- smmap/buf.py | 47 ++++--- smmap/exc.py | 6 +- smmap/mman.py | 249 ++++++++++++++++++------------------ smmap/test/lib.py | 22 ++-- smmap/test/test_buf.py | 7 +- smmap/test/test_mman.py | 3 +- smmap/test/test_tutorial.py | 40 +++--- smmap/test/test_util.py | 51 ++++---- smmap/util.py | 93 +++++++------- 9 files changed, 258 insertions(+), 260 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index ef9d49e46..66029cb99 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,12 +10,12 @@ class SlidingWindowMapBuffer(object): - """A buffer like object which allows direct byte-wise object and slicing into + """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. - - The buffer is relative, that is if you map an offset, index 0 will map to the + + The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access - + **Note:** Although this type effectively hides the fact that there are mapped windows underneath, it can unfortunately not be used in any non-pure python method which needs a buffer or string""" @@ -23,12 +23,11 @@ class SlidingWindowMapBuffer(object): '_c', # our cursor '_size', # our supposed size ) - - - def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): + + def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Initalize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access - If None, you have call begin_access before using the buffer and provide a cursor + If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size From that point on, the __len__ of the buffer will be the given size or the file size. @@ -44,10 +43,10 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): def __del__(self): self.end_access() - + def __len__(self): return self._size - + def __getitem__(self, i): if isinstance(i, slice): return self.__getslice__(i.start or 0, i.stop or self._size) @@ -59,10 +58,10 @@ def __getitem__(self, i): c.use_region(i, 1) # END handle region usage return c.buffer()[i-c.ofs_begin()] - + def __getslice__(self, i, j): c = self._c - # fast path, slice fully included - safes a concatenate operation and + # fast path, slice fully included - safes a concatenate operation and # should be the default assert c.is_valid() if i < 0: @@ -91,18 +90,18 @@ def __getslice__(self, i, j): return bytes().join(md) # END fast or slow path #{ Interface - - def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): + + def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. - + For more information no the parameters, see the __init__ method - :param path: if cursor is None the existing one will be used. + :param path: if cursor is None the existing one will be used. :return: True if the buffer can be used""" if cursor: self._c = cursor #END update our cursor - + # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): res = self._c.use_region(offset, size, flags).is_valid() @@ -114,27 +113,25 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0) if size > self._c.file_size(): size = self._c.file_size() - offset #END handle size - self._size = size + self._size = size #END set size return res # END use our cursor return False - + def end_access(self): - """Call this method once you are done using the instance. It is automatically + """Call this method once you are done using the instance. It is automatically called on destruction, and should be called just in time to allow system resources to be freed. - + Once you called end_access, you must call begin access before reusing this instance!""" self._size = 0 if self._c is not None: self._c.unuse_region() #END unuse region - + def cursor(self): """:return: the currently set cursor which provides access to the data""" return self._c - - #}END interface - + #}END interface diff --git a/smmap/exc.py b/smmap/exc.py index f0ed7dcd8..5e90cf722 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -1,7 +1,9 @@ """Module with system exceptions""" + class MemoryManagerError(Exception): """Base class for all exceptions thrown by the memory manager""" - + + class RegionCollectionError(MemoryManagerError): - """Thrown if a memory region could not be collected, or if no region for collection was found""" + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index da6fd8153..6663687d0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -20,36 +20,36 @@ class WindowCursor(object): """ - Pointer into the mapped region of the memory manager, keeping the map + Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - - **Note:**: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" - __slots__ = ( + __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide ) - - def __init__(self, manager = None, regions = None): + + def __init__(self, manager=None, regions=None): self._manager = manager self._rlist = regions self._region = None self._ofs = 0 self._size = 0 - + def __del__(self): self._destroy() - + def _destroy(self): """Destruction code to decrement counters""" self.unuse_region() - + if self._rlist is not None: # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted @@ -67,7 +67,7 @@ def _destroy(self): pass #END exception handling #END handle regions - + def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" self._manager = rhs._manager @@ -75,41 +75,41 @@ def _copy_from(self, rhs): self._region = rhs._region self._ofs = rhs._ofs self._size = rhs._size - + if self._region is not None: self._region.increment_usage_count() # END handle regions - + def __copy__(self): """copy module interface""" cpy = type(self)() cpy._copy_from(self) return cpy - + #{ Interface def assign(self, rhs): """Assign rhs to this instance. This is required in order to get a real copy. Alternativly, you can copy an existing instance using the copy module""" self._destroy() self._copy_from(rhs) - - def use_region(self, offset = 0, size = 0, flags = 0): + + def use_region(self, offset=0, size=0, flags=0): """Assure we point to a window which allows access to the given offset into the file - + :param offset: absolute offset in bytes into the file :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed because we reached the end of the file - + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager fsize = self._rlist.file_size() size = min(size or fsize, man.window_size() or fsize) # clamp size to window size - + if self._region is not None: if self._region.includes_ofs(offset): need_region = False @@ -117,91 +117,91 @@ def use_region(self, offset = 0, size = 0, flags = 0): self.unuse_region() # END handle existing region # END check existing region - + # offset too large ? if offset >= fsize: return self #END handle offset - + if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) #END need region handling - + self._region.increment_usage_count() self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) - + return self - + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - + **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to un-use the region once you are done reading from it in persistent cursors as it + to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None - # note: should reset ofs and size, but we spare that for performance. Its not + # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - + **Note:** You can only obtain a buffer if this instance is_valid() ! - - **Note:** buffers should not be cached passed the duration of your access as it will + + **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) - + def map(self): """ :return: the underlying raw memory map. Please not that the offset and size is likely to be different to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole file in case of StaticWindowMapManager""" return self._region.map() - + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None - + def is_associated(self): """:return: True if we are associated with a specific file already""" return self._rlist is not None - + def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - + **Note:** only if is_valid() is True""" return self._region._b + self._ofs - + def ofs_end(self): """:return: offset to one past the last available byte""" # unroll method calls for performance ! return self._region._b + self._ofs + self._size - + def size(self): """:return: amount of bytes we point to""" return self._size - + def region_ref(self): """:return: weak ref to our mapped region. :raise AssertionError: if we have no current region. This is only useful for debugging""" if self._region is None: raise AssertionError("region not set") return ref(self._region) - + def includes_ofs(self, ofs): - """:return: True if the given absolute offset is contained in the cursors + """:return: True if the given absolute offset is contained in the cursors current region - + **Note:** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) - + def file_size(self): """:return: size of the underlying file""" return self._rlist.file_size() - + def path_or_fd(self): """:return: path or file descriptor of the underlying mapped file""" return self._rlist.path_or_fd() @@ -213,32 +213,32 @@ def path(self): raise ValueError("Path queried although mapping was applied to a file descriptor") # END handle type return self._rlist.path_or_fd() - + def fd(self): """:return: file descriptor used to create the underlying mapping. - + **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() - + #} END interface - - + + class StaticWindowMapManager(object): """Provides a manager which will produce single size cursors that are allowed to always map the whole file. - + Clients must be written to specifically know that they are accessing their data through a StaticWindowMapManager, as they otherwise have to deal with their window size. - + These clients would have to use a SlidingWindowMapBuffer to hide this fact. - - This type will always use a maximum window size, and optimize certain methods to + + This type will always use a maximum window size, and optimize certain methods to accommodate this fact""" - + __slots__ = [ '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window @@ -247,26 +247,26 @@ class StaticWindowMapManager(object): '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles ] - + #{ Configuration MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion WindowCursorCls = WindowCursor #} END configuration - + _MB_in_bytes = 1024 * 1024 - - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): + + def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsize): """initialize the manager with the given parameters. - :param window_size: if -1, a default window size will be chosen depending on + :param window_size: if -1, a default window size will be chosen depending on the operating system's architecture. It will internally be quantified to a multiple of the page size If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default will be set depending on the system's architecture. It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, the manager will free as many handles as possible""" self._fdict = dict() self._window_size = window_size @@ -274,7 +274,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._max_handle_count = max_open_handles self._memory_size = 0 self._handle_count = 0 - + if window_size < 0: coeff = 64 if is_64_bit(): @@ -282,7 +282,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. #END handle arch self._window_size = coeff * self._MB_in_bytes # END handle max window size - + if max_memory_size == 0: coeff = 1024 if is_64_bit(): @@ -290,18 +290,18 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. #END handle arch self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size - + #{ Internal Methods - + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :return: Amount of freed regions - + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. If the system runs out of memory, it will tell. - + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -310,34 +310,34 @@ def _collect_lru_region(self, size): for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and + if (region.client_count()-2 == 0 and (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region #END for each region #END for each regions list - + if lru_region is None: break #END handle region not found - + num_found += 1 del(lru_list[lru_list.index(lru_region)]) self._memory_size -= lru_region.size() self._handle_count -= 1 #END while there is more memory to free return num_found - + def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, + """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" if self._memory_size + size > self._max_memory_size: self._collect_lru_region(size) #END handle collection - + r = None if a: assert len(a) == 1 @@ -351,40 +351,40 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here if is_recursive: - # we already tried this, and still have no success in obtaining + # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise #END handle existing recursion self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) + return self._obtain_region(a, offset, size, flags, True) #END handle exceptions - + self._handle_count += 1 self._memory_size += r.size() a.append(r) # END handle array - + assert r.includes_ofs(offset) return r #}END internal methods - - #{ Interface + + #{ Interface def make_cursor(self, path_or_fd): """ - :return: a cursor pointing to the given path or file descriptor. + :return: a cursor pointing to the given path or file descriptor. It can be used to map new regions of the file into memory - + **Note:** if a file descriptor is given, it is assumed to be open and valid, but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths are preferred unless you plan to keep the file descriptor open. - - **Note:** file descriptors are problematic as they are not necessarily unique, as two + + **Note:** file descriptors are problematic as they are not necessarily unique, as two different files opened and closed in succession might have the same file descriptor id. - - **Note:** Using file descriptors directly is faster once new windows are mapped as it + + **Note:** Using file descriptors directly is faster once new windows are mapped as it prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: @@ -392,92 +392,91 @@ def make_cursor(self, path_or_fd): self._fdict[path_or_fd] = regions # END obtain region for path return self.WindowCursorCls(self, regions) - + def collect(self): """Collect all available free-to-collect mapped regions :return: Amount of freed handles""" return self._collect_lru_region(0) - + def num_file_handles(self): """:return: amount of file handles in use. Each mapped region uses one file handle""" return self._handle_count - + def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) - + return reduce(lambda x, y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + def window_size(self): """:return: size of each window when allocating new regions""" return self._window_size - + def mapped_memory_size(self): """:return: amount of bytes currently mapped in total""" return self._memory_size - + def max_file_handles(self): """:return: maximium amount of handles we may have opened""" return self._max_handle_count - + def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size - + #} END interface - + #{ Special Purpose Interface - + def force_map_handle_removal_win(self, base_path): """ONLY AVAILABLE ON WINDOWS On windows removing files is not allowed if anybody still has it opened. If this process is ourselves, and if the whole process uses this memory manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to + closing all memory maps whose path matches the given base path to allow the respective operation after all. The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep + This really may only be used if you know that the items which keep the cursors alive will not be using it anymore. They need to be recreated ! :return: Amount of closed handles - + **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return #END early bailout - + num_closed = 0 for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: region._mf.close() num_closed += 1 - #END path matches + #END path matches #END for each path return num_closed #} END special purpose interface - - - + + class SlidingWindowMapManager(StaticWindowMapManager): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles + Once a certain memory limit is reached globally, or if there cannot be more open file handles which result from each mmap call, the least recently used, and currently unused mapped regions are unloaded automatically. - + **Note:** currently not thread-safe ! - + **Note:** in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than + create more memory maps (as the open file handles limit is hit) or if we have allocated more than a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" - + __slots__ = tuple() - - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxsize): + + def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize): """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - + def _obtain_region(self, a, offset, size, flags, is_recursive): - # bisect to find an existing region. The c++ implementation cannot + # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. r = None lo = 0 @@ -495,20 +494,20 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): hi = mid #END handle position #END while bisecting - + if r is None: window_size = self._window_size left = self.MapWindowCls(0, 0) mid = self.MapWindowCls(offset, size) right = self.MapWindowCls(a.file_size(), 0) - + # we want to honor the max memory size, and assure we have anough # memory available # Save calls ! if self._memory_size + window_size > self._max_memory_size: self._collect_lru_region(window_size) #END handle collection - + # we assume the list remains sorted by offset insert_pos = 0 len_regions = len(a) @@ -526,29 +525,29 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): #END if insert position is correct #END for each region # END obtain insert pos - - # adjust the actual offset and size values to create the largest + + # adjust the actual offset and size values to create the largest # possible mapping if insert_pos == 0: if len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side + #END adjust right side else: if insert_pos != len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window left = self.MapWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows - + mid.extend_left_to(left, window_size) mid.extend_right_to(right, window_size) mid.align() - + # it can happen that we align beyond the end of the file if mid.ofs_end() > right.ofs: mid.size = right.ofs - mid.ofs #END readjust size - + # insert new region at the right offset to keep the order try: if self._handle_count >= self._max_handle_count: @@ -561,18 +560,16 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here if is_recursive: - # we already tried this, and still have no success in obtaining + # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise #END handle existing recursion self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) + return self._obtain_region(a, offset, size, flags, True) #END handle exceptions - + self._handle_count += 1 self._memory_size += r.size() a.insert(insert_pos, r) # END create new region return r - - diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 01f6cc918..67aec6333 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -13,18 +13,18 @@ class FileCreator(object): and provides this info to the user. Once it gets deleted, it will remove the temporary file as well.""" __slots__ = ("_size", "_path") - + def __init__(self, size, prefix=''): assert size, "Require size to be larger 0" - + self._path = tempfile.mktemp(prefix=prefix) self._size = size - + fp = open(self._path, "wb") fp.seek(size-1) fp.write(b'1') fp.close() - + assert os.path.getsize(self.path) == size def __del__(self): @@ -33,33 +33,33 @@ def __del__(self): except OSError: pass #END exception handling - @property def path(self): return self._path - + @property def size(self): return self._size #} END utilities + class TestBase(TestCase): """Foundation used by all tests""" - + #{ Configuration k_window_test_size = 1000 * 1000 * 8 + 5195 #} END configuration - + #{ Overrides @classmethod def setUpAll(cls): # nothing for now pass - + #END overrides - + #{ Interface - + #} END interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d3e51e2ee..d07b7f4eb 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -3,7 +3,7 @@ from .lib import TestBase, FileCreator from smmap.mman import ( - SlidingWindowMapManager, + SlidingWindowMapManager, StaticWindowMapManager ) from smmap.buf import SlidingWindowMapBuffer @@ -22,6 +22,7 @@ max_open_handles=15) static_man = StaticWindowMapManager() + class TestBuf(TestBase): def test_basics(self): @@ -82,7 +83,7 @@ def test_basics(self): max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), + for manager, man_id in ((man_optimal, 'optimal'), (man_worst_case, 'worst case'), (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) @@ -114,7 +115,7 @@ def test_basics(self): assert manager.num_file_handles() assert manager.collect() assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index cc5d91488..d903af681 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -15,6 +15,7 @@ import sys from copy import copy + class TestMMan(TestBase): def test_cursor(self): @@ -101,7 +102,7 @@ def test_memman_operation(self): fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index ccc113b4f..5c931de73 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,7 +1,8 @@ from .lib import TestBase + class TestTutorial(TestBase): - + def test_example(self): # Memory Managers ################## @@ -9,76 +10,75 @@ def test_example(self): # This instance should be globally available in your application # It is configured to be well suitable for 32-bit or 64 bit applications. mman = smmap.SlidingWindowMapManager() - + # the manager provides much useful information about its current state # like the amount of open file handles or the amount of mapped memory assert mman.num_file_handles() == 0 assert mman.mapped_memory_size() == 0 # and many more ... - + # Cursors ########## import smmap.test.lib fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") - + # obtain a cursor to access some file. c = mman.make_cursor(fc.path) - + # the cursor is now associated with the file, but not yet usable assert c.is_associated() assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to + + # before you can use the cursor, you have to specify a window you want to # access. The following just says you want as much data as possible starting # from offset 0. # To be sure your region could be mapped, query for validity assert c.use_region().is_valid() # use_region returns self - + # once a region was mapped, you must query its dimension regularly # to assure you don't try to access its buffer out of its bounds assert c.size() c.buffer()[0] # first byte c.buffer()[1:10] # first 9 bytes c.buffer()[c.size()-1] # last byte - + # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). + # into consumers (e.g. struct or zlib). # Instead, either give the buffer directly, or use pythons buffer command. from smmap.util import buffer buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - + # you can query absolute offsets, and check whether an offset is included # in the cursor's data. assert c.ofs_begin() < c.ofs_end() assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the + + # If you are over out of bounds with one of your region requests, the # cursor will be come invalid. It cannot be used in that state assert not c.use_region(fc.size, 100).is_valid() # map as much as possible after skipping the first 100 bytes assert c.use_region(100).is_valid() - + # You can explicitly free cursor resources by unusing the cursor's region c.unuse_region() assert not c.is_valid() - + # Buffers ######### # Create a default buffer which can operate on the whole file buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - + # you can use it right away assert buf.cursor().is_valid() - + buf[0] # access the first byte buf[-1] # access the last ten bytes on the file buf[-10:]# access the last ten bytes - + # If you want to keep the instance between different accesses, use the # dedicated methods buf.end_access() assert not buf.cursor().is_valid() # you cannot use the buffer anymore assert buf.begin_access(offset=10) # start using the buffer at an offset - + # it will stop using resources automatically once it goes out of scope - diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 745da83d7..745fedf2e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -12,18 +12,19 @@ import os import sys + class TestMMan(TestBase): - + def test_window(self): wl = MapWindow(0, 1) # left wc = MapWindow(1, 1) # center wc2 = MapWindow(10, 5) # another center wr = MapWindow(8000, 50) # right - + assert wl.ofs_end() == 1 assert wc.ofs_end() == 2 assert wr.ofs_end() == 8050 - + # extension does nothing if already in place maxsize = 100 wc.extend_left_to(wl, maxsize) @@ -31,34 +32,33 @@ def test_window(self): wl.extend_right_to(wc, maxsize) wl.extend_right_to(wc, maxsize) assert wl.ofs == 0 and wl.size == 1 - + # an actual left extension pofs_end = wc2.ofs_end() wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + # respects maxsize wc.extend_right_to(wr, maxsize) assert wc.ofs == 1 and wc.size == maxsize wc.extend_right_to(wr, maxsize) assert wc.ofs == 1 and wc.size == maxsize - + # without maxsize wc.extend_right_to(wr, sys.maxsize) assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - + # extend left wr.extend_left_to(wc2, maxsize) wr.extend_left_to(wc2, maxsize) assert wr.size == maxsize - + wr.extend_left_to(wc2, sys.maxsize) assert wr.ofs == wc2.ofs_end() - + wc.align() assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) - + def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size // 2 @@ -66,56 +66,55 @@ def test_region(self): rfull = MapRegion(fc.path, 0, fc.size) rhalfofs = MapRegion(fc.path, rofs, fc.size) rhalfsize = MapRegion(fc.path, 0, half_size) - + # offsets assert rfull.ofs_begin() == 0 and rfull.size() == fc.size assert rfull.ofs_end() == fc.size # if this method works, it works always - + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) - # with the values we have, this test only works on windows where an alignment + # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions + # We only test on linux as it is inconsitent between the python versions # as they use different mapping techniques to circumvent the missing offset # argument of mmap. if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) #END handle platforms - + # auto-refcount assert rfull.client_count() == 1 rfull2 = rfull assert rfull.client_count() == 2 - + # usage assert rfull.usage_count() == 0 rfull.increment_usage_count() assert rfull.usage_count() == 1 - + # window constructor w = MapWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - + def test_region_list(self): fc = FileCreator(100, "sample_file") - + fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): ml = MapRegionList(item) - + assert ml.client_count() == 1 - + assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size #END handle input os.close(fd) - + def test_util(self): assert isinstance(is_64_bit(), bool) # just call it assert align_to_mmap(1, False) == 0 assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY - diff --git a/smmap/util.py b/smmap/util.py index 44e94124d..137c19d7b 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,7 +12,7 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -__all__ = [ "align_to_mmap", "is_64_bit", "buffer", +__all__ = ["align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -27,6 +27,7 @@ def buffer(obj, offset, size): # doing it directly is much faster ! return obj[offset:offset+size] + def string_types(): if sys.version_info[0] >= 3: return str @@ -37,7 +38,7 @@ def string_types(): def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. - + :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" @@ -46,15 +47,16 @@ def align_to_mmap(num, round_up): res += ALLOCATIONGRANULARITY #END handle size return res; - + + def is_64_bit(): """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxsize > (1<<32) - 1 + return sys.maxsize > (1 << 32) - 1 #}END utilities -#{ Utility Classes +#{ Utility Classes class MapWindow(object): """Utility type which is used to snap windows towards each other, and to adjust their size""" @@ -68,7 +70,7 @@ def __init__(self, offset, size): self.size = size def __repr__(self): - return "MapWindow(%i, %i)" % (self.ofs, self.size) + return "MapWindow(%i, %i)" % (self.ofs, self.size) @classmethod def from_region(cls, region): @@ -86,7 +88,7 @@ def align(self): self.size = align_to_mmap(self.size, 1) def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, + """Adjust the offset to start where the given window on our left ends if possible, but don't make yourself larger than max_size. The resize will assure that the new window still contains the old window area""" rofs = self.ofs - window.ofs_end() @@ -103,10 +105,10 @@ def extend_right_to(self, window, max_size): class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes - + **Note:** deallocates used region automatically on destruction""" __slots__ = [ - '_b' , # beginning of mapping + '_b', # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages '_size', # cached size of our memory map @@ -117,32 +119,31 @@ class MapRegion(object): if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot - + #{ Configuration # Used for testing only. If True, all data will be loaded into memory at once. # This makes sure no file handles will remain open. _test_read_into_memory = False #} END configuration - - - def __init__(self, path_or_fd, ofs, size, flags = 0): + + def __init__(self, path_or_fd, ofs, size, flags=0): """Initialize a region, allocate the memory map :param path_or_fd: path to the file to map, or the opened file descriptor - :param ofs: **aligned** offset into the file to be mapped + :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted - :param flags: additional flags to be given when opening the file. + :param flags: additional flags to be given when opening the file. :raise Exception: if no memory can be allocated""" self._b = ofs self._size = 0 self._uc = 0 - + if isinstance(path_or_fd, int): fd = path_or_fd else: - fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) #END handle fd - + try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size @@ -152,8 +153,8 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): corrected_size += ofs sizeofs = 0 # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will + + # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) @@ -162,9 +163,9 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): else: self._mf = mmap(fd, actual_size, **kwargs) #END handle memory mode - + self._size = len(self._mf) - + if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping @@ -173,7 +174,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): os.close(fd) #END only close it if we opened it #END close file handle - + def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ os.lseek(fd, offset, os.SEEK_SET) @@ -186,92 +187,92 @@ def _read_into_memory(self, fd, offset, size): mf += d #END loop copy items return mf - + def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) - + #{ Interface def buffer(self): """:return: a buffer containing the memory""" return self._mf - + def map(self): """:return: a memory map containing the memory""" return self._mf - + def ofs_begin(self): """:return: absolute byte offset to the first byte of the mapping""" return self._b - + def size(self): """:return: total size of the mapped region in bytes""" return self._size - + def ofs_end(self): """:return: Absolute offset to one byte beyond the mapping into the file""" return self._b + self._size - + def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" return self._b <= ofs < self._b + self._size - + def client_count(self): """:return: number of clients currently using this region""" from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 - + def usage_count(self): """:return: amount of usages so far""" return self._uc - + def increment_usage_count(self): """Adjust the usage count by the given positive or negative offset""" self._uc += 1 - + # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: def size(self): return self._size - self._b - + def ofs_end(self): # always the size - we are as large as it gets return self._size - + def buffer(self): return self._mfb - + def includes_ofs(self, ofs): return self._b <= ofs < self._size #END handle compat layer - + #} END interface - - + + class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) - + def __new__(cls, path): return super(MapRegionList, cls).__new__(cls) - + def __init__(self, path_or_fd): self._path_or_fd = path_or_fd self._file_size = None - + def client_count(self): """:return: amount of clients which hold a reference to this instance""" from sys import getrefcount return getrefcount(self)-3 - + def path_or_fd(self): """:return: path or file descriptor we are attached to""" return self._path_or_fd - + def file_size(self): """:return: size of file we manager""" if self._file_size is None: @@ -282,5 +283,5 @@ def file_size(self): #END handle path type #END update file size return self._file_size - + #} END utility classes From 1cd3760be374521a7fafe2262e58c1703b71b9aa Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 22:37:15 +0100 Subject: [PATCH 0276/1790] Drop semicolon at end of statement Fix pep8: E703 statement ends with a semicolon --- smmap/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index 137c19d7b..394e6b197 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -42,11 +42,11 @@ def align_to_mmap(num, round_up): :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY if round_up and (res != num): res += ALLOCATIONGRANULARITY #END handle size - return res; + return res def is_64_bit(): From eb40b44ce4a6e646aabf7b7091d876738336c42f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 19 Nov 2014 17:12:38 +0100 Subject: [PATCH 0277/1790] Added link to readthedocs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8c9ae4251..e15c9bfc5 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ The package was tested on all of the previously mentioned configurations. [![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) [![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) +[![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: From ab4520683ab325046f2a9fe6ebf127dbbab60dfe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 19 Nov 2014 17:19:52 +0100 Subject: [PATCH 0278/1790] Added readthedocs badge --- README.rst | 9 ++++++--- gitdb/ext/smmap | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 194e24658..186218d1f 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,9 @@ Installation .. image:: https://pypip.in/py_versions/gitdb/badge.svg :target: https://pypi.python.org/pypi/gitdb/ :alt: Supported Python versions +.. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest + :target: https://readthedocs.org/projects/gitdb/?badge=latest + :alt: Documentation Status From `PyPI `_ @@ -44,7 +47,7 @@ DEVELOPMENT :target: https://travis-ci.org/gitpython-developers/gitdb .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png - :target: https://coveralls.io/r/gitpython-developers/gitdb + :target: https://coveralls.io/r/gitpython-developers/gitdb The library is considered mature, and not under active development. It's primary (known) use is in git-python. @@ -52,10 +55,10 @@ INFRASTRUCTURE ============== * Mailing List - * http://groups.google.com/group/git-python + * http://groups.google.com/group/git-python * Issue Tracker - * https://github.com/gitpython-developers/gitdb/issues + * https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 28fd45e0a..eb40b44ce 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 +Subproject commit eb40b44ce4a6e646aabf7b7091d876738336c42f From 5e3dbccb05f85f178d3260a27df2f59a69221668 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Dec 2014 23:09:42 -0500 Subject: [PATCH 0279/1790] Bring gitdb.test back into distribution --- MANIFEST.in | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index b14aed9b3..597944fd4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,7 @@ include gitdb/_fun.c include gitdb/_delta_apply.c include gitdb/_delta_apply.h -prune gitdb/test +graft gitdb/test global-exclude .git* global-exclude *.pyc diff --git a/setup.py b/setup.py index dc142c518..1d327eba0 100755 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def get_data_files(self): author = __author__, author_email = __contact__, url = __homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.utils'), + packages = ('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", From c38bd19706abe5cf0bf0e7b3e9ad2b3e554d28ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 13:47:19 +0100 Subject: [PATCH 0280/1790] Increased initial size of decompressed data to obtain loose object header information This appears to fix https://github.com/gitpython-developers/GitPython/issues/220 , in this particular case. Nonetheless, we might just have gotten lucky here, and the actual issue is not yet solved and can thus re-occour. It would certainly be best to churn through plenty of loose objects to assure this truly works now. Maybe the pack could be recompressed as loose objects to get a sufficiently large data set --- gitdb/stream.py | 7 +++++-- .../88/8401851f15db0eed60eb1bc29dec5ddcace911 | Bin 0 -> 222336 bytes gitdb/test/performance/test_pack.py | 3 ++- gitdb/test/test_stream.py | 13 ++++++++----- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 diff --git a/gitdb/stream.py b/gitdb/stream.py index edd6dd2b1..b0a89002a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -100,7 +100,9 @@ def _parse_header_info(self): :return: parsed type_string, size""" # read header - maxb = 512 # should really be enough, cgit uses 8192 I believe + # should really be enough, cgit uses 8192 I believe + # And for good reason !! This needs to be that high for the header to be read correctly in all cases + maxb = 8192 self._s = maxb hdr = self.read(maxb) hdrend = hdr.find(NULL_BYTE) @@ -243,7 +245,7 @@ def read(self, size=-1): # moving the window into the memory map along as we decompress, which keeps # the tail smaller than our chunk-size. This causes 'only' the chunk to be # copied once, and another copy of a part of it when it creates the unconsumed - # tail. We have to use it to hand in the appropriate amount of bytes durin g + # tail. We have to use it to hand in the appropriate amount of bytes during # the next read. tail = self._zip.unconsumed_tail if tail: @@ -284,6 +286,7 @@ def read(self, size=-1): else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) # end handle very special case ... + self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) diff --git a/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 b/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 new file mode 100644 index 0000000000000000000000000000000000000000..d60aeeff74d2983bcc2abd03163d2cc8c422f819 GIT binary patch literal 222336 zcmV(zK<2-A0gReca4t*`reoW-ZQHhO<0ScF+qO<@+qP{xIkBDWzq;OiYieq`rn)cY zrl0C}Oza#@h*(%ySedzii2012O|1lA%*;)!jUDJstR0Me6?kAY$rk__!v%mWvr>c|!t=BR(^2i-&|PE)rDF zfk!^fRVBjq@wl-S!7GHO6AF3#{Ejj{^|X?hEje2mmunc@?hfXE+wMbLFsSn}W!6+7 zrYYW(|IjgGY6HhALjLvWd9t14+jh1M1wY}9KMQ@OOYlBu(6qS&Rz zG;LAnQ9Rj@lNGbDc1h^FSj3=9lCpHf@k2234o%CG%}5xx#lpPiD%5;uG3Xu2Yr(BE zon1Lm&JI?IPN%}Rb~b9`J+dXv>-8lU_vK#mm(pQQ7@{r{q2nMeA~qv@Gw_PqX;EKR zYJva*$1;8Hs{v+8yqVOgCAu|<0k-zn#_`2qrpm?Y)|1sb9eYEQUIOo7(7eT7IW>rD z?E~Qg7s|}D(Xf6qCW&m3kjSFF{$`C%`>Mr%1$O^_v}k3|KGQ{)D8SLPAkOhWPZ`cC zTr<&Dd2M?Xsd*_}V)+fm|3y>!`!?`BmM%X#f18>Fv0%gV(M0m5FYVjc^TBhVZhN>B zu4pO}qYsQ01JyGW&j?Y^0~ts%-Lbxx#!VtmBwzj1&3VWV+GKHGO(qNvzo#vR{&FYp3$w zc4GpwFu}I_h6%VBt7ja9Tfabd+ShP7$JuqZ4NNSYs=uu z8wE?Q0BA=a`2`jP95%&u>kb;CbZ1m4k@}HhSjcNEx3x!DVY}MHHLf!|bJZ+Em$9b~ za&;zCbTC$oV9E|m>Db+qog2;$-Bc#Zk5GW8B!-f~&+h2pA0L5N?ap_KsrSmh5LCus zCy`swgR^0vIWd*ul+cD(_e5zPdfG)xJJ1-k0`oygR|SN+^@J7@q_VR2Zz$AF>TcG1 zjg&v}aMCNNokyrbrU61s$f0t##PrL(?V`#ZZECh)u2>uZ^BN6doe+HpOO*$L#od20 z*^|GFj{XSQugs$!^!hZ#Ei#Y}Y|CfljmV~fgs6_L_=0xNY&btn3wP;sUJ61y&{5X$ z&tMgRGaF_77+>&!0GUI#G?SYC!wqn$p2-XzDA1k!dq)a>jgj!*2gA@%<7$#qnE$5HR4@*Evg7r zkckvjr^Kk@L}OWM&w^2fn*4Wj_B&+qFl`2fIX?VXcNltJq&X9*LwxMorn43jK}QBZ zSq@8!6<~+-%Q|Ni-7(!Wy?w32Q;A5C2BN*ntp*mcU0%DGLrmZ?f^HVXle;tEtj;J$ zH;rDQG*EySc(1En46dl$@+Sp^V&=StZtZ!WNYvg#|`S9#W5sVWE((c zcH3nuNGKume`Uz5gYrv)7;X!?f7h4+wIsk5xfVHM$7eY-hS`oJukZG z(SP#jJ^tYN{p%IuZuhqA!8iHIj|eueTaw^53?dVZ+*Q4Yq&!P`HA{2Y{zrQiNkd)S zUv^UHM*_q9;qDXmcocr41!2Py`O|BVuqH>)aHXsAi@k+OyK zBMsWy(Ny?X*J;T8;y2g7>E<{2hq392{kJT0(&E>@>&G7oh%8g@^WgJ}_&tU1>Z_Ie z_0J*M#~+b^*YRbk{kk`}-l~J1vtB}P(WS&z+tuRgiGnP$@XZuF8fIs8!X(j9$sd8Z z*bEt5XKkv)3KpX9FFAx02yQe+Le#Z{s)leVmu`wiEj*3SpZ3a?lh~2BTMuJo$1-0(qT)fX#lj^aI!=B?59>;u@NwcZ!}G-inz2$f5yh&#V% z!=CF&xAS!Aq%V;#LL98lQZM({K0Oszz;E{GSDv%L=kHek-bbzE13&qD zg^tCWDWJg3LKBfUk?-aT;;$`a`!)4~ZkgDFT>9N4ikrVoFZLd(<KAEZ5If zgsFSkRZptUwr?HYJd`+iPctHk^neYis^(M41lYT+*T)`{$2Tw?qD$I9E_K6NPZ<{b zW#X~cuG87WRVItW?SjU$wJOXRS^k8L0?e@X@_24$wXLYeKCFOWzlMQ~Gt7;T?m$?Pt6|Lt2uopo}W538|(i;XW-I~~7f3i1;Bd;T0hUU&~6zbK49vNj9sv57vGv3|XQ z^vu=O&=NEDe*5S=ttkqA+DZ2gG{(Qie8)~!j?#x!hIH6X)!ODi*J6*Zdgz}BhXO*W zFgqx|O?JZhjUiq^gZ712i)4pKTgB-K+BWmGZgm}H%TwQw^8KDjCwSMz*z$>W4Dd5I zOTAhgY`#RC>9z<^qE&zH1%H1ULVS!a&Wh>@NnHtOTL{e^6J(HekjUyM<=3uO*R0*5 z*YMX9d`&_BhI67!3-y{_E#kO>gA;v*XVePvp}`ZkqPEjurgV%2FkYvH9dl z!_8}ET#J4lNS(Si5jr9^yQmb?Fzmc2#Fh9w*YuIeoc(Rs^|X8zluh;c{$lLNc>HB7 z{E?&+;}EL#zlORSMC^Kzcox+7Az<*qE^>;He2(WrQ+!qjUBi*JOi0-Y1b65Oo*Hew zljY~fn+2)E$BQeCX}`2Xb{Vv-W59@5=ZdR#=m5_h9f6(SdMk_T8q?SR+nWtKGuO%8 zInzB3N%*zRTXO6^c1!lVmb&Ou#%4MigUxC&ox%M7b2!qB{sobdvPuC`0s=BK1{nV@ z)b;oeb^p`P!%Ta)Z^fB+E#WBf3Wr5bAg-+3JM>NzVyrc=S z6WKXBomuGkBiCkTGLyJJ!W&p9#o0El_Y8E^1=pV+I$hYxY@9Zkwm2;?Oci1INw|ka z`r?Dv447~fJEU(sNPF}P4F6^FA{j`NN4u$k+9q$9)g-Ir?j06JX~=5-S3?U@(~# zb|GVqm&EDHKE*2vh$6Ry>-x7r$|}PrUEPJnvtLU?P87dd1)4Ol&P>a4owNEXTt|bu z>e;y{uey|dID~k`XN*9378)iMtF+KF1Zib(B`4TYsQ$^1uVHVWVO$3NxV!X1Nr6+% z>c=YA_?LHy#l`Q5U5p)uTB=Td)yBP5z!wlHc3{SM`Lj#J#-t==qbTLWWC#G8*EZRb z#iEu->z;KUwS!dR0u?SBoRov;nv!5Fzv~@YV@J55Rb@jk4!toXz+wY;$j*PJhI~gu z#{~CCRMQ*nRU0`bjECEq-^k195jn=PM33EFo&;7CX1<4RppoMDdqo^g>ws4bxlW+( zgS+)=b_r=GH{i1-24BCcXmdZ$%i&+B4O2&+P$+{IJFf*5nAShRoh2-z(Soy1AQmTl zH&6p2wYDS6^`qy+^K+ygG8K>A9u{t)a<&lyfQ1=F(HGw_q?1mr;v6XQ7_0CHZb~=-IfVOZ0LNKvfJkZ`__*P-3aO>MzP!_GMe#$L zk0fF_RA2p#d#NS!a}-K{e>xBl0WIgrmf>pNWn;AA+ONdPaXxhNIE1&F)~enU#O;)_C?Lf^_e?O&eq6I;xuV!cL+sy8m*^d)^j0 z-N&YbCphB?inuKB-nnf7MUi%#zA4=zRE`@d8$|(_y;jv4=88PyfO=}4^?NA>M2)8j zW9B=a;hVo0Bqfw2IVY+^b?P%G;VJbfVODxAT@Dp_?9`W!NmK5IvO&&l3ElDU`upe0 zZ8y_MCU#EgspzjP@ItG8o`@ESy21}9a(Eb-QJS}| zHgCv>h2JIbjR@SpcusNqQRKo5>XlUo>;tt&Uo!MVB#&;x4(5x#Oc|`5W2_gFg~dKR zX-rFLWm@r^@M^1=or+AjEvWxi$u{rNAKp?_P2zq{meK+NZWA#SI5`)ot(yZ&tzy(y z;sH?Aa|s&g;3@@J@(Uq^2u*9oi|4u}FrST=AZ&Ut%_RWOB&c>GpKPuwcmkdHj=x-W zTDPuS4p~YVWep-Y*%YVHcQc7Mf8KS`6>fO2372*j&gf8`y$NP4DT5kp%> z(0W>4T`@YgpLi;8y@+be?XycShpWi6CjAcs!D=aa2Ic>* zQ;gT>i^zv%#Cj%cIT;2x5mN!Tl=OF@VN*Ho(D5I<+Nq- zKt$IZdN>4S%ro_EE>6iiUM~FWgv*qy08k^X(UmEGfQEBg1DY$QV|(W93N379e&5D9 zpji!szoxOQ7$2wQaq@q`q8A*C5vD=x&;dP|u*UnkSr2CQya>11($=f4)q%O^rS z5=FT@+?*4OE;39f>YM1gTw6AspaPMjFlE(amcfG~(ugNu12q-#W3B9Qw5{xCgLs+_ zfrRyrhO{3papdP&>&dIv$DV;P-Hs$-XI#CAQ7sBJz;a-zlw?5s)&}yPQU-xe0EL{w z)3@dIAb(7{vEr#H#QORIa@;wI0o;*iqmB$VJEj^&;=F)W4Pt`fNr?d34*85c59i)L zkXgw^jd&Blnas@4NT+tm1?RE+TWi8>hSGN+v2YRrTXhnm0o9A5vb7{L@Jv}nnIQij zY5epaAlh7k!#vRW4L!K|XhiG7J>x%uz8%Y+{{W9ypyINDjUm5Y*3A975X48fYTZD? z@wE!gBATVEEg>;D@A*N5(v+G4Zg~~64Me#SZ0TpdR@OYG-!p0oIQp_w^_10;o}MUt zXWd_APHhXZC4LKpN9vc=qLJH+C6H-ufB!L68wS`rD%`zw{NawHt<-%&T1s_>fUzuU zh$sQ!T|@v9kAfa$S*jY{^K?FwP-{E5h@j#+3hH$C&F(aGI_C7tsQ?#d)zIDP(3##I z6P;^BxMdZED6PgqYo12aQviwzv$&|Xs9)X>o1Ify54CGNZ*PARp9;U-Y7*!bj6Efr zO!q|c_3d?@m=;Y=^i;xqNC8ENt(iGiW>yWJ98CKp)F8Lz(K&j%Q>i4KM4rx^-bh5o zefNc&Pp(vxxen?Ois*nq(#T=6C77D`6tv}(ga4>AQ(ju*mF9{Svk|%b%~!2|<0{d{ zA-uy{jJDj=RiMhke`la9!nn&2_7Fl{^_Y3_w*iN$(B9n%VQO5($H;00Hy4@NAI+JX zWi&4vH8H1Fnl&1)rqJhDdGif0ZqXfw@bmSrCULRGD-;R-|)d)2}moJLRZ* zt=#K@&Tpe}60U-0eMyNV<+of~a}l4P4^njtRxwG`vv;Q<<1XT1APHY*(;z#wWz61Q zR(@T}d9TCyvLU|fw>dB!IEwJ9Gmrs(5WUBw1=%qt4>#v2eTV4}1ZC@;r_#3&1xG=g zSPtOAP+DtrMr`(jtb%~V^Z4yKs8TQik!AL?r;2|cS z5TfN_UDQsqCS{l^Y>KI&R8YlUf=;k&IVcm6qS%GaW-WiW|Mf2Kp1 z-2z77*hmR6Wr<%qm`r7wDwjaRi|8gv*GR)X1qo-UU2ZMK6jcp6BKh*!(vAq#ZAB|o zcrO7xW88Lm`Znm#`P1gTB6h&G*vS{syqVWjY$!8v(ARvbx>~cAspv~{W|zTx+{L&O zw&_qf?_E*{!s)LF)5hKw3?Uzv9o1yKi`H|ZfO=#7+N)ke0bLm_lU1);Rf)nDuFCnE z(ekU1^n5pFGdahQZzD#MpWDNJiA|@rW5|nnVhXsppnkgPo476Yk#&!lWwyXrbNslb z<&e2-#o`R43@emNZ67ePcfiU-)=j%f{F7Jp*~rHnm0NXnS9BZn5ue%d{`!l5;BFPE z`XgBQcpvD>DC=co&l7~M-ey&x@aHenqZ6GAW~~2l^?As})Otf+j;0ofoBqs?A21EY z#;Fy6x-&iFG7t(QtQhNEl5&K_=Xg{yBFK0NdI@y)C%pmNYF%!5;H(Fye`Fk$EVAzB zaBEvb*NBG#b8XUE>Yv4~lFO^PU!z(-AmvC|Y-6+WJ0N-HV>TQgy@gA~%^}J$2)KnU zS+lt*+Eanj3R(qe*8L2M3Dr`zEUNEW^sIx#mXG%wwLrD#kC z+Ci?O22Y- zuNYVBZ1B0=o`#|Aufn*@F&n-+x`M>+!Mo_2d3P>FL8zAKH1dENwnB7J%mlSpgIrSE zO8Et3$LbUXJitZU&G8pLp6CQ-L4{Mh>wZ2e^a)JsT~-f#0{-IhH($A5cd9ACr_jJC zY+tSoXz_)n6-iK7Bekxr%K){ZI4eX{GT33V92TPcni1Z^Nz(Io%873|Hy>~Y@(j|) zMtB_!@y7b&pMp3}n?j(w2NjAaXRcuWR>r>=90s@B+%oDO&6a1ZS$fPhDMHS|8w0(H zqVE>y{DY+NBw2+i>1|E|F$hMFMR*9}K>Xf^CLWQCFb;;ZB^wN$0 zEF6tRAcQD;8&`BEiReYc@FaO}>(hw!+TK+n;Lx;N2DHtzVloEb-S_puwV7H^y|2dI)%f;zj$ z0flpF!}1JMK6^LanpNo|MjS>Lg-|z^d@@om4MdiIcQYj-Azz%z(8gvUO9(I1l9bez z-}>w>(Ljn^_2ce5kK<)<_c~MWFVL>fZTQtW&l?GA6q04P%rkTf5{r$8#0BXOulW34 z-IW!;=&td36f4>J4e1`oKZQM@9_M(JAlzn~kmscF9UxvH6Pf@WbN7`KjHgdS2C->T zAQ2ap!Df78*dvUvzw!Vba4cx=k}AnA(?&Z!t?~~%4Exex^MUf+qV{CY7aJqfq-=({ zc$S+2ckG<(<btKP-7^jmT89<5y9l|3ZH-3HT7hi%N^Q zGJ<08m*nlcxuJR>G+}m<4-oUQC&TXexM3Wx9!M|B7=OLH$>yU)Bq&~64s9`%WzFoEiYz(c3E9#y2nkE571~0IXdCR zhW1C6a}VrupK!f;gI=6pQnlYN==bRm*CfC4gpG@LW#!ce)r+0D@1KdmJ!~OCO%I&H z4Hb+P%`y%Jmnq&b6!e9tgoVE((4S`G*;B=*dW|kt8Y?fcJID?|sPf|2ov4+{)AgX; z*wytF^F&C~NETQ#t!Uk0im0^=Na&Ey=%eAO@{rFg#2MOcg0=#prvfN$Hff*};MmO2 z^Kd3;0sY8nhTw7#IDqAUEtO3=#1q6bR#easG7Cd9x^A6^aWhNQ(6K;<)uHU*fmh^W zJHp1ZG#Q6dYGp&Xoiuby)P1UxS>Ym2Jfs`7gU&Ys_Fg=09Zugcae*9vjYLrl&owF8 zE%S6^3vT_;jB0r=rJ&euE6R6KNzvb{WLuP4lg3h&lPk>0RixQ7?XK;e=H8;1h51}> zFeo8hcr*Mw#l)@Mcn4HRr25qt{w2D5pU1-MaoY zYVE?c$?&m-I)Az3*TL>Xw^2(qe1D_f7MMm;4~22xWn#E3DriI~nvGr$;n}9r_q^2H z(MZqVFX~A~mo!Gzw|87lDddorAlAj_hSeIJ*6&tS*|NvJmVyb|E_Wx`Bg1T(J`xi2 z0qfK%-KSe_Hm$kp0lavCLXTdsfSNu$>??(^etgGg7Rf@wm||JGM31P}cGrwvu5_4` z)@GiuMsLZ>Ox*@eqFXM`Z4#C{9Abr0!!8DU>}TDgG(q!(qpEK$CveNuKbC|s%onPW z9@>5kk6|x&TnQQWEQ=a_5eZvkMzv#G?E_)Kj@6keR^sk5ti;^!HBdM=Zta2k zUO1?~M%!;%YpfrP3=kNmP`U(?T7%Y`3K#7&4EFI)l2C^F2&kW>{n=~I7wtA}JEmkk_q7wag19~GB7E7WqD%0=7&M`ubvICSxYoUgglzS)0}{4c z;`TnL=o2M>0x62xz`8xX^ECEqXa%jm<0&iW3q+zN$L091^HH6cv9{UUO-8rz!%#iw z<(<$`+y~+GXz9D4(Sze<2Zd<1X-pmGEA@+eRcya3O~_M?uL|>+NwbFWxG0Ea6C&ld zZe5XI{Bn5_da@ziq%i?lR1~BpgXL3qNBXc5#DuB7M}(P}3JaB8EEI&59TwStZ{h`J zK1gUH-`b{i2s>)BGkDZuty2rSq%pULW++an46Slp_A)3QX%v>d-nd`$a?c8oMo~?_ zuxpJ%QV0?Fuy)>ZVR)W1B$dpd#Ez(s75NHYK`lj|;q7sfVgjL`-iR>M5%r*J=Z|)j zW=NJXKAjW!!(qa*9kg>cp7Q4FF#{H2WC%^`7a-CMxde^1V8M$bXTXY|F<@C(LA(yfh<3qcBB zWC(=o+h4%)#b&{i%d@=LSmeh(w8oNuYv&xKkLf-ktdm*T|LtnLH|uzL-z6 z+Qbe1+qs*f^zzC2sPYGZPWR##x?4R%`vuxKyd=t&14m!@^M(@O(|nd|xRT{ON^8XK zsLvthgndCgetB-0bgq}I*yG^^&a3J#1nKqL4A>dQl~e|JqlVj&8mBr6G`;GeplWP0 zzG?gb+x4XUF;+@IG8U=I@^m8l%(cWrL214d%b=HZr+IAJx`Z;p`|HSfY!C$^|4ITq zH*|138Z3AgNng~V{~ne;2-5lrT@~IDY9wuR{pm_?(_gWiQv>Qa6?)G6z0SEafXjzpfqM4RiZCGCK% z?h)eW5C1_nN42n``T>6o5@0tC${98lv=AN?bbv5Qz@R9v; z1SRjV3=VQ!$K5Xrju=6yBbCj*#9+?vF4?8xM7ljT!hjosxB#m5w=3w9^(az(yX5<0 zx<7_^#xUTD)zLn+5PJY%4VV>DRDHP+z?OV;)Zl}W7=HIR1A3kB$2cG zrE?BnMID4{yP1=S9GIEV*aScaB)-2iAjvW3aWZGi9R~Nd|1wT-Oug!x>B*^l`NZIs z>%X#kD;lv1Rs$VjYq7Gj1+q|^>G2%10=viuT}=%XGcT6bwdeo@#6}#vSK)Ynz`T2o zQ*oSp!poM?>a}}C>c5nNKvA|#KxF(y8RZb1zOQ-4E-%R!f^bpfvHB9eL%>5v${qiZ zq_FIw_w*ux`Jd#%50_PhlzgJO*Jxt@dPw^cEuT+MNzFIF@&Y~U8U}Q{-exBj&>ebT z7na6~l6J|b3%Qdb5SCJ4ADrBjd3ekDcSjloI%}J^Zq`yIa{qmJ!Qae8y&ol`_u)DR zoWB!)a+p@w%r>$vm4c2cEVbdOylnkUV{u=zb8G*Tt3W>v=XGM44*yVmeLdGF&tJYF zlxjm|#nI1~fY0NcT9zN=kqrh>mmdnj>$Ek*Lf}m3)iikdF$sqBb+(#AsQM^F*oTg^ zVoxLbH&PEBIkAx28E+CV@)TYhQYU#buq5&afa=ZTv-kdPM`SJH>f2FGgO(<@o|HgH zQ9XG^zIdD}N|pxIhL#}-E8A8cZ=(2N>vo@N4M234Glp^X4%^8)E6d{}FNo(SNnvAj zSUg~+wTvGR47TxbiXyU_#F}bud_NTTxW0|c-^x6a3?auIuWork#=5%H;l$ZJCv{D> zpnm7#Jc*~(An%1kT-cU)zxA&OzI*{xsSgNvj(hg(- zd$(TS_!tRxphYx4`-e+Bkxu^1NymPI2(J6hy`2|Le>T2o9s>QgWM&%j$(xP_CQ8Dm z>Mnt3cbtd$F`-XJeHq6~ z>w@0(3s0s$VGOsQ1;CjaG-*e*0#SUD^Dh_ffB5|7g08X4eJ9|94+Q01hLd^aOFdh! z!L1k+97}fla{2oaWC_`S2OuH=Jce?MS*U!=7YA2%euL-&j|nP;`#WwYL@Wd%ph^+e z`idYF7o&6=Ja&_1Sy;L5$|56}Wt4C%!AptpDDPy*LrSWI>uG?{*K6=YG>K-=*jCK9 ziU~vHR!)f6d%gLzM9~JXKXi0E_DT){E&YwjfqogA?P^Rw_Zu^Dnlytt9_sXGrPM<^ z7{3@$3+?EqPRnck5omvvmT%f{KmYewU-Uu$OD8Y6PxrLH0hHsd%oTGRnkXfeWkgp! zL#khG?}yRUp`Ct$X8rH$Bi3KuU~kLg_H&9Q$&@_&y=ak2@c4#xO{6f{2>4$Ys54GNnD$OLH#PI>KZ!`0o7*)tus(Vb#kkjP?W6g_ zext61NOZd&=2QYozG`XN$VLtKv!D9+C_*Hp-;$tX)tQj3_fhUzM&#pUnqYW@a|(*D zg+^ihBerzJp!gNZ`J7jL378H{oUfxU9bO9%zf*WKP{j0`9;M8ppIFm#vtW;QhJ*l>d;(xmZn+$%RVzV&Ug(;B8UkLK^H6#rwJB{(!bN2)E2Ulgh03Yf05 zbnzw@Jrar`JviY)&LB!~qj?mh1Y_>FTt`G8!*q;$xy46)hp{mstF_x70&)UdILXCq zCj3}y6MnW_xS~w2gi#3|H8+s%bPpK>+KcIC;Qn$XQ!^@9H3jN3*r+eVK0JCQL$u-3H=D z-;r&j6-Y7OlnUqM+q_t+hV%@F8GMYm`Xv#ozV`;}&D=!MpD2c4HCx3G!_8<(8>rr< z6Ga|prj0?WkFn8nGam727_od%LsO52!GqfNKSr_+v58fnj`Vbg zBC&jX5g_?SA{r%7im%@Bm(^H3$MPteAS6oi!m5@~`|hAsKNs}UGzgrVb5oK&HTO|I z6RS41;}eOXq?W=}zH1Zj1FSHsF=cwyqEx^5G4QHLeqi_LV z+m@=?kyYq*02~0{vx&Smk@*f+n9bDmcHo($)mAPiqN`0Yf;jt@;a}Ggc>07-BOJ_A z3cPRPdjNKN&d5Bj_}-IVb=CpkWyWAz3p-WYZ!%kyQq5?lCmPtgwbv|+=FAOie#OX$ z@e6BR@b}~Q%fA&n8@n6wVt9Fsk6i$9BVe(&tzW%(uO`8^# zBpadH`z2L4l>s?tofrREU3(1L=U_q&Ns3%^-Z?mv<#P&FLf6-q$Di_nvv82aNwz-0 z*d@k}Z_*hirj^X#R-JJ}_wxl2J;)NpZ4+WM7J=~H2$czRziy}K&tUUEXP^=CyW~M@ zjuJ*lUT;QHXDfLTao@j!!Wmy~Jv*z+zb%rShQ;cbupSTUE7ssfGS1C8Ns#kNa7HvZ z@4UnfNh?puFAa)?@ix7$%`<|^p}H$}->qBg;8=jDYkXeU(UmNFiY3W5vl|^5 zo~*R?Nu-X;m8Pr>2=fI@+;)V?zLUR*0|Yfds3|en6x<*nd4X1~r{!_lS1(DtDWp#9 zwm!`f-C2Us5W2m<;a$<#FcHBPAU(FS?B$7__6#5#kFb2ys^WI@YI_XLsf(ycdloX> z=C#2$wk=Kpz;QD+jxJ0gYL-zl$@&vTE~t0rWN8>ils~tDVM_!X3l~w+Y*L+fcf2o5 zuRbp`na|}K-dDUA*squc%~6P|-*_XWpKXsy&ATmpq-ZJ-j-A&NhM`!l0;sEIle5Bh zZyJ5N`}&&HOHyndSMVqpll-t?*LZrmm?h*6^OtsI3JDvnELro2c2{V#i-bP=po;HU) z84$<>dSTg06s~c`0jPsrYqH0}GRayJn2$ka7Ru+ZyU}0QzE;KhZ07hegq2+6J;1!} zbttp-iwT7FjW$Wb{a$T}xbzcxdHAG=QSWtNzg6d@&-ut@d{MWd{(~U?9OwCQS zi^<|K$oPFaDPis#m0-vy$NMTG*!4qd8S&2*2F8-YH+fg`2-V%CF?DNI>z0FEs17=O zf8|e6^2n6Z0;j$u$>^^<|NGdy{8ak~+ID#hXHT#6&5ju}rY;+-$&lgGVd@5Y^Lm0u zjkKh=+SojtwP6El!89YX5epRX9bPGf6t6Ku#HDSSGCL(~u4lmRzv+cQv*}6JC`Y2$ z+twj7;iR}oD0ZR$huIus8CR}f9iL>Qe<%{ zCe(7|$Yp7*y5;D?hZvz!l*-IC`w|^05J0ee?nLD2m`J=tF_HkHB~bEWlH58GDC^V& z#t*et6KxYuT5xMt=l_P$Wa9-&re&gWj3oiAj&Mqrgm5}e<*4lo4L1<2EXf@g9am&W zz*62<^X|ILc_1ei1CH+*2i;0H*80|#<4de6S}c$QL^<6>a9Ne%m7q&IFw}oG}$1)vB%(52IZ#gXWrx4Mg!zzw(SCo{fFXZH z_OJPTHa)gQ^F*g2>|}#?;l7wGNf&Q5pFveA_9cGKG;bL2 zvSJXZBW4KBGEBQLvXRmd(5?BeC=nbR%x@k9W`)(VX_p$CRpU;Ecs@66G2VE`!yrs( zZARG`mPoPn+aDOg`>bprX*60Go>@mvQY$?JxmGZEHlSN=^1Ch^6C~6n+`p8c5qy!j z_FmPm;YP{@-Gk6gwlAatTB}M=)8U(+s6B4|RVU#UDOA>k%E|MVX=0rH>VapgF*RQk zul2&rfCJ<+Lm|4w4~3Zr7s$FqPJQy<*_Y;vyr=+45CM}jgqI7M6ajJekthIZ^vVc- zxSu`sI_tmdz{Na^%s)$1&zvqUdCl{hr`mUOrbLw>*QHLXPzgca7Dc+TGiC>{Q->eF zC$9xtlrDs$j!Cl?|Bk6=`n}X0j-&)en)LV6QR^exg~>5li8(xS4jgl@QKl<<2DD?i zux8!WoyJ@H;5JX42#DJEeXtGKnZw1N8e(8n5ku%; zsFE#RKpdmr;wMJVMbPi57K-QPrT$}n*3i%8+n$+%iGTCQ_mR~MP{F@Pb zmdCo{F8?xV-GA!9&UBT7(0pJ5J^&G*C7<27GG18UjWa|$Y-dxRg{`{^d38Xabm)KR z-_XGJ*neDs&53d>MpT7iDeuX)M-B^x2Mt3jKWJ&=c||!2mWqo&So4wI^<1mG0K^4j z)jw>?XFCKnVe=~eZaCTr$H;Y;`*H-`hBEwoV?}@@;E{!clPDAl00xxc9?s;W;~GeI z3E{_AKLX(~g{%beeyNrwr0&!mfgccjaUFbT%cq*$pdvs(tFNOc_fra0raxi)nDBvo ze9ojm0`sM~r(fjW#hCVHNE!l7#n$Fzt_0XyK$W^=v?3Qv>eHfo?+kq|^U{qnMXa$k zRk5g$bZuzTjtVcsEPT<+HUKKY;mQdzyUWC>#^>yEfVE4;k(dMI{#@b1aHBp5SjA<; z5eV@#nVl2DL|TdfJ@88)&xQdqG-2)MVn9aHsPbJIIYkpKlLEfaWS#sRPdQ|?4<&4h z?AtVM&YoOK)-dKA4jn_*M(bd=y82%=N2=2f^|OIrN*ukkXzd#C+5)c^p_8R&334zc zs{W8HFf&9K%sngv&ur4(uXnGxr^U-6e#ejP2>Kms_yWzhQgIH72L`sM2V#90Bm>Qj z94gTO@5nd2Ji>rTmWkr;a;GECfb&fRXK|Mta=^x!m(HHP;s~vq8 zX%886G4P?P7Fw-0$WI1rHw2N>aiE-$9d*N?#pMt<(51vwV8pZ0?6v&864${Eo7u=r zhA%7moRje1;x$Floj2fZB6u>pFucIJEm@OEAm%FaYR((ptn{`oVhA|Bu(2s7&E%%L zdAC%eQ9_!%A8jv3q@TNXpG`(X8PJ=Fs@GTz^sXg`qya?OZ0XdHO9icqP+0&nZcwZo znb#XNw*o}*jpuR(`^(p%H7XbsP1&S(gT5T%_<<#(aps-VpHP+^&xpgpGt>Fn6sFwF z1ddJF11kR}M(2mWY5%)a*YvBWutXCUSQJLcbDA#j#fMGeDd}o3B|mo$Z5E?LJdJhV zet+nz0g8WMg9@Tv(r`nvbG}d4iD}%8PGeWnE=_A`43{M|>lVlDh}pZ)1O2AZVdiX` zWyi7q5@v5e0(!O253Za=`H>v6r@m=qx>$GQZ&P(|y!`P5nr4Zm|FrG&;-bZYGDH>+ zzykMj@zZP}@hUWSZB!2Vu8ZZycsloVK&S?3xiUxYSz*pKD)s>HRH{yhFOvv|=>g)- z-jn!=Z?^n){^i}hH9&F%SZK?0!1e>*c!Yf zC7~mE=nd`Vy>j#LV0M!mm^q^JON=(P7{(9_l1{B8I&$J&M@aR%Kvgv`acmRr7D{$I zzYz+za}6c&qrumFv02OPXC0N)v6s$_sj__C<~$icOG13nUMU3xC2q>c6v?j@NhiUw z-BT||u!7z3rvU+YV!N|GT(c0w?wCo};g$veyum8NmUt>uMpcr`UC+{>eXzaOPkcq= zEOkS8OYu!i>^ke?jRaG~gvwie1TA7)cgMi01gC7O*mA*d>zRPn9zczdoCG2YQ7Mae zuX;0MGuYnZ{>h^b2U!s%Us}*{xF>a}{xc$l?tZst0TFB47hvav1>t@0i2V%k!i;c? z!Gah1jsZJCeU^c0BZK`Ze6=bTYxh2pz-N9hIP&oF4*Z$bQxdkT+7vyOn{NP=y8DaG zLN(5Ad~+kaEd4Q*l^PY2GTj`ZE)~FKPn5TlnT`{fyAtsWWOY1N0*4MsCBM662S?23 z&O-z-jyS}|Cb%I9_jmPv6k0v!$vb(=!Elww`Msr6E7?maJFFtB&yc_G1Ov>U`Y(J) z5VfPts_Kw?JAnqOtT0a-Qdzs7<4_(=f=p#&21Xq-Ys2c$JBDY;rrKG`6^axSS)}=& z$qFQ`skDH9)^e?ggWOyFAnJkm;{0}+__cg!Bbt&H{gS5lN2c^43}+zoH2+PR^DYMjf5}uB|7Tk8aR8C|ql^^m-bx48t5>E?kKz|S`L90frG-uj z{k#d}Q7o$5xlN;=gw&BKM(`jcOvkDcz^~{1svwtz*YJ3RdF@?LRzog&;)o?k=34*V z^Z_F6Yg(&BLZn#*RTzhZ+>dHgj3@JQe?efPaDwZT@+F!bSi9uYrXNVjip2fllm+oC zWaKHdYuWmSSx{Enisb;twIVYQ&$Xb!jcoB}t0g79ufGigLge9eHjkQC4FO#<7X?9- z3e+%~(vY$?gru#6LwzzxUZrn-lu($7`rA~N7DR{|dK`0DWH(+7(NiLsr%OyXw;qer2Zl#QYfA zBG7^5CVBVYN^~IrKG&}cWGXv%)um(`%W932Znt|c(Rvnvl=Od!f<`--5yR_zD#fu| zGWj;cOikav+%2N1-gwVtbRlmlSM==Rk4u!W>UvStXH`V3AKetF{4BeBuh(_JZ1q%d zi6h-8DdCIK47SoeK=a%cJM)mZ`RmsO>gpXE28o6vhnQ_;03$%$zcEI`P0TyR>i6?Q z63uVmT@4Y8m>7=bw8qdkZU1ZiBH}k3iA^ul3LK1HjGL0O3_c zEeFZ#{T~1)K-jYfHjWL5Bti_RAZrRJYIRqE&NE zti6!LbtHUgS5A<{@GEhSEHuPy(`tFZRbeEv5r?{u{w(awDND<(<8p z3iadW!qnEt_Jb0INzJPtjYXfwHu(F02)!3k6}*0BfpbUK>lD9d7e2ORspd1tgxM7c zp6=GJc2mu8$VG`#i5-bSB39kwf4f2c&G?~(XBAS{OFZoqD_qfo05iZ2iV#q1)M2q} z$>n*9OId2{Dus!5Ia73tutV#{`k;(a$}V3~CHw^_&U4In5G0SBLQB?ipa~c+n~t*-<`qWH?{%&4S{hUbXl*EJ*1|@)%M>yTx*l zXQf|eeii}t81)^zxVQMFm5gmj4@~{yOdL0FHYxXXatB(nwS*!ur9p5~ zv!phE;_GuDJ^nA~#t*hF#Jt2PtQV9Xo7Ltw_@ITTmz|1#|3|dSOH??`TbqcF5IRG8o{3Ia`3=76f0tF zD*9=>s3l;KLR6-)8ng!>iAAdxsJf6*YIz{yGoZ_r6IN<|Z(EfXeV!f56}zEC07Z}} z*4io}T|IB@f`ZVWHM$K+D(j(H5F2!OM$?t#14^YuRP_X8cjl7c&+hXAh1uk#1Bm!T z+PYwwie4=x^QF>4+m>WZxAgJgTkOZAV@|QvHPm$fr<);Lgm3DsQ^hMb_~mfNI)C6B z=Sp7o_X}eZjWgbFh?n%J@5+`Ynm;=s1UxjU2Im}F`4`ncy`jHT4QA<@G@1LRl(mp z=l44o4o25|tH(dy9Wg-|pkOJ@<$$ns#-~mk z<)JkZvgrso8s7XTY!Kr-T8SSUhY?G^w#(07gNbEb9AAYDOKGJH6CA)WOF1D}Bkn(0 zTa}Y1%}m8dslst*fTTz~q(8i$%0Gax0=W3lNdkppJPScm30)?Hs9gLYR?ys{jB3UN z8Jsm%_r-PfypTc8GMbQ}3zMTjcp8)nuXWpgg9wH%&8X43rFBv7XeZrzxqi;zIs$P2 zNq9K6QII+>%98Sr<6#TW7VHL4w|4MY$jP7d0LwZ`_yVx@N~>slOQw`{E_enevu)d}ck>KRG zBKU%@UdsXB`Ik~4t5%SrJFb}6stg<$z%yCfMQ=@`a9QZ#II80bJWFRYCW)fLz{HqV zD+HXqQNvz9_<|lS{)=54`t7X{qk+K_JOVz@=Hp$kOY_^cw zNBc#c*d&Bo4vBoI>qe~>qZna-oDGfWuEb+9K-pwrhCX(fjJtMI3v1LrOoTXYeYb(& zVCh$JaRM%Nj8z+c-%K9Uxv$0<)M!+?_+vr{b^Z}9It35H%7c6EbHcyEk9Cl!sw)mE zo#xm0Y)UauodiA%gtH;7Pj`zK&`PA5xcYYTlCl`iSrcAa?xlmvb6sP|=?{F;eAe&9 z3gtY;b&%yu)6pRIt4Rt>dsNHQ`h|9>)Jsf$&0hAd#fU!ohk5HkWu-4O!rU+! z4)#7KDk>`f<(Y8#Ui%xDcK5wygaN>tV; z6iQVBfD6!#K+Q^DdB$rC(yo7JJLEA0EsNxUGe8dQ2<^g)Qpu;t>l>=^8oM*=DLt#U9JD7Bi;)}wQSa#R%CJii<@;pRY)@Vbdh&$ejXl<()-TNRkOlaWaK8@i zzhmQV&|VMK0qFtuk`J|__WXBzweZBSU+UTScqgff&WyL4(oC%|ME|oClYj15K!50p z?2OG9h=siIKWE;c;FNl{5XB{m@04dk5Fp(%qD6z6)cM9QaUs!PtKQJtKwB*ptOYq+3b1r6FFsEMG1h4T?K)4E3|cE$fGjK4TtWh4zCoA7x~bA z*cO%G+Mg2OSZO*B^iShSj1)kB?t*k^WrmF{JRViBE39*xF$lVm(NyRpp}wWwo%0_YW#DrWwz{ z_-H?pl-8d)s2&6?EMR+^iQPpoFs# zc;(ejuxn=s1PwBFc;aahr)q8%mNVTd+-PKtRFXQ99-AM=w|p&@XGV*f!-ky}Sm=V155sKwkd<$WUt5;CPZr+1n*9Qd} z0Xz~@sa6=v3fqUwPRj$0t@70u`K;%_?lq#UA79ErD0#?~#UG=uV;M_V?#tf)0kueI zTiJPhntKQxPq0N&QmftnByfTLys$IX6HK9Jq|V{ubjsZ!GtIDQQzS<sZiE#-Gs+SFpCsm+ReROlu}`Ah$n#r23akYS~s0)u!Rk7xZ2=Cn8N z%Fg-T7b1g}Q;8R{xT4VAOCPy*tDt<*PN=3%|Kv2o+Arh{Ablr1=l<<;xXYv)jVG&T^10Z<_aEL>`GDQDO654F6_ zKSImVIXK6yNxSgoQLgeAYyiyQrGBo!lL=$Od_y)JPrqrZ3FP9V3W@ZpG`3w=YGUp8 z0p)IPY`Qw^3-JE)5UjuG24@rvbQ(U?RQAUITT?k#&lX%grgoEe(T<~P_L=w?uI>_vvR@7mpdPMJP6}X zUD=^xj(ybJwEIpDz)8T=m zsE{oW#QlOFIkBb?40~r`jBMje<;a7(wICs|h=-y~Ga-Nyr(bl$s*X22TTiUP`(*#L z?wY>*hUkDu>|?rjiPb7fIBE^zeTT)#I}`cQwGeyhgng;0S3zl?Q`ZY{13dk06j#Nd zN~|36O&y$Gq7}W5kkmk4_Azft2FcTM8KTr!#3Rz*2PTqsUMhjDQx3-7(RLYRjc=#D zz-Ofkk+OxN?kbh*WVGE&Ix6JCpk|-(S8U zHyKe)qX=ns^Zk+w5nFJPn?tnE(ORNo#aS)&L5YtcCt+HvRn{?MB41}s=zMQ`ge(Jd zLPVK;fY|c+DwK!iZ*OReBUfPBce&9wHF-#*g2)5fGx(CGy?kRd&d79E?$en68zTvu zvAJq;YtrpK$h}x%o4MV3h~*WdXc4oC<7D)Li4c#Ed-?+aFMn|57dZhg45)+@X0gIi zH7!T3l7ZtrM!HSA=v^%$rd<+ErYWuLR|Ao96+(o%IXINcAC!=4k1LM}|Ma5!JekBf z8}L|}v~s_iMc=)A%KiBC_PPKEs${m7GxCxWDRA$n!yv&?l_;23PFbcW7$`|)h%*(_ zD^25CjjNR{K9wqY%ax1IFk{%1Ls(Hi0oqggmw-ZJF}pUeA47DYa(yY1(MLmmLTLGD z6iJlfP~rRy&ssh5-@+m0h6+T9YRgDuSBF&g`Na%F{(67GH0ISRHm?z5Wsu^@R}-R7FdGk$aQWEA!?cI3olq`k4+s3Z9Q-x51ZPhP zc_9-qI9p-wm=+j#b(!sY826dycR}rZ<^p7xbr318zJC0xStcnGyz3>0;nh;a33Lah z(maU^DJ87Z?6WDXnPE|Fh+hx~RiXhFTn(51}%-n~!y?Nn*6lCPI)b2p%cjeMMm9Rb>j4HMJJ2loA3Yvn8lFZv2ME%o^6^)jiWGq0WgP16 z#x|F?O|1ls3|zlen_CoFsy{_F>W|#SkE=L*KGOHw(*@o)6Ch4C@!}R^Q-8K;NMn@w zini6M4R7#l(CX9> zrCIr6Ri;rr#4YTRl_G96u&&ap%6qRu9MkOsEET#4p`*n~Uen3F$YvoaG-u{t4x3kj z$CP}yp~?wI-)fh(EzmZ3D5dsHC%2^WNC<8+?fr$I+0_M_G7-ylhl0ZmeOrXg_zdUo z`?gQPGZh{<-LAxmQ3by+7}c4)&Tbzybo<9u=MqibJgeB=*Z~?)!`UQtti_w^e~LB* z``y~>rjhn8a6N_LzG=ly#=Et~foIx{6?}-Prw8_~=G6D=o5cX8_|F$z?nrYY1?{P95%JMUjvkN1{UhvT;PHb5QgDtnk-3_r3>O>TWQEetL zApbj^J_o60!C24NTk-$B;pY`sqQ ze6g@yA?j8LE`^T$Cl!m0`}jkWU)uC0piCgq6B5LlbAGt>Jg5t*H{GUC*E&w%1_F#^qQ=#M82)sL{H<6+$F7q&Us z%WJ45^qlJwGAEW3P*a+VXgojTqZDUbls=Grm}ffvA9AiP5+S$!#7zV6aIM?JLs`Po zq+?B#@$@*3ZjcVR#_=^k(2HBK=wXm?69A6YzC&$zy04E7XO_i563RsA+;r><=V0`y zV7~%(z?WglW}QA&TaznJUSElDMP!F%6eTH#b?A)O<6U55izWN>=l8IZ@y0}(Sutbg zU%=d3@$_wzC>XtUq>Y7RtdX^=Ae!N+M<}Nik0N+M+oS}kNbq#faG>BHu{SbpAKTQp z`CD;=2k*V#Ej-nK#oQaydVVwuz5{oT%JPWHos$FM<@u6$f<}>iuD7~D2a&uikI-TdUiq8YpfQ)&sxwUPq=pCKTTg)O(jOnCG zDE`z_mhyO_Z8u%yW+vdJOs5~McUcsMf_CfPyi`0Pw3{J@-7}4Gfd}GFf=o_1+*1B~ z-#do&S$Wi(4R~@S>DhA4_Rcyo6QG2T$(tFa2UV{Uvac~zvVl~ThNvQJNjNQ0SSYr0 zOU!I>mPo>x3eT;09_{eJTQ+Hn@J5F~nn|fWo`zGkEq@tef2IxU)B< zIUU!zzj`y=bQQ$$T^`LIAWSk;u=-T+Ix6;;hMUau^0|GDdD|U(PH5L&*P3BDh0%%i zUHe#4uY`cp@MB7AZnJ~OVkda28_PByiO&pxc!F=?RL#mI@Wm++RQg`%BY4j#g7ufX4N6hvs zjbz!@Bs@JVmsz#z)RINPa40v(Rt{3wSgr+9bQHQ1Ej=S4pJG!=-BhTnxNip;A~Q&E zK(c=3l_`|t8*Dzni&9mk(e2{lyf#O?#~ry4K^!FPdVV~nZb6lG+ff&wuxf;d7Or+@ zf!*3-;qQUi2<5bumS#IV2|}@wdm3eUOGn#-6b_%Z<=Ez;V}T5>J1d!fQHj}8ZSA@? zq&SZt?<}ndDjLD4m@EH7hZ2QmD$HoH{-_%X(*6HCycU&nn4eB6b*fY6>U$}UTnzT| z3)%Hy=~e?wK9gAM($#T4>uFjEC>kR~=%tk@#%2Tq<8J(?SDj(QtzSy0 z?fAS4XFvm<*3=_z`H^7-SZq5;eClB7)pSSijE>MI6cZc#@lDbi>f>QVkfc9;Dw zCBk#HL3_q3x9dr3KLa`;4W=&pmuGz>8~QVh#sIDJ$y7QF-2_qT^|n{=aeo$jnMJ zsfNbN)5R7u?!~Ckdmyavv8UOMR-BR0{ytb7_iKRAxirrvBWt8@@dNmaUQ7?7& z+OaaQ0rhttrak|inB*izsMLmAz1aT@=87zPR2MJZ ze>az$7CEc{KstA*He~{wzq$u<;E)zkeoHNp_NI}@b9Yc*I$qHm4J0B2WF_!-b)5z$ zrB1d#DW%3aIbCoEIMhRj>AI9T#9FG;F>Q8@Q_?Nau5(2^-iQFGwdar@YHjrsc^ysf zl20$mmF|olEwBF5SL{Nmlco+lMkQ3_%mppk+Bs@ymmcy%zi4lf5$M_(U|9VlXLdovx;p>{6db%-_SGgf7!Uve2 zqGS&V8r0D(AO= z@9l!-*>m!>;tz2evX?&DqwmsQviV96mRMGVN}pKbF4DY>_lv5t7c*F&n->pLo@o^_ z^eiVAQ8ej~{NJ-3ede6jS9}+hoDEc_T`HM}fu1@L(D9DrU|)7m3#MJGn{JWoH|%w7 zj$7NwIz*&qd_+sAOhePjJW(RgOaSeF;_c|f4*^XHvjmuwk9hNiI*Umw5#f#ml5jj= zmvb%u%Xv*N{xDJ@;zjQcOBP~ejaZt0;r`(Qgi1|Wc0`xEsHz`3IQZ^D-Y;mBBYaDw zZjFH4$&K4~R_)gl457R(@mS{4;gE)d4#KU*f`ejfOFjWaAHfaO9x5xX80Mj9(}b&> z%)ED2N}#cWPdgR1FUG#+jlH&Cx(Tyfhd{U%?3?!TgM&wSn~;RcoVb%{)#XKBcR&(q zz02*^0VMR=uE+Jd>Xj>{KG=19HJv`~nhD?*vZDn3*Bw$y-%W-_(F$qGRp?*od6?A$ zWGBDc5hGk?&KgP|k`2?-xK7ddZuX9ALhY3d5|}S05;Ymutn5b^|4S4fN4#8$r^8&k zMT_IAs@elsx0n_>$MX#>$RP?Yu1#TjqR0gf{@WcN??X@WVYcK3pSOT=&=b?r!hjCy zJ8v8$?fTdiBEdmG%iafJ3gg%+MjAQuI*25FaiHP!g=2)?Yc|BPjJSsW?OrJwIkcS@ z8fQU)+9Fjg?&pHQMH-EC2a|0rAR@!68Gnlb(I-Mt#^@l*$U&IAULt^9O)WSw7Uwkz zMTR`y3&6GEfqN-U!OHokyf?mMAk#NugXrYiOB*r^4ITKiB!@yWB(uGGe)=_%;dg+VTf!l}r&#d+gQ)`*%7YD+~jZ1sBKv1AVjo0sK_ z$7pF{i#3YSm7GP1WY=U9UUs)owh>>yg>B&xMwd8feb(QXWL&>f*NEcm|5vgvplWhFUz9N=c597RFf%jaVgK4TZ3XOd)Kqizp zH!l_2?y>}HG#cO0NB?B#3B?AiIxH~}wfw_``zN0cE3qoe$3vEag+z`UhfCK>4SZ=$ zO#n05H40&}zCPpP?W5`<4sM=e#Za_8sDlLL&d%+Ge@9geiC z^6B?Te4fy7p0w<2(cZDT{q&G|Mrdr?05tqZV!BIqUDjxd?8in|0Z?S4MzRNLzPSBfdNA zEg)d(;&ACmOi78OoQ}n-;Dz>|&F4fv%Z9gN9ZOa12L?KjV0^s@rT3nozD(UUohvxRD@9sa{&X5fO3m569Q%~i;xQOwlGV*DghZU2#=TB80^D% zbOTQVby-;LdlG-kyo_=AZgpTd93V0+A>#&{%zSAV+}{EjZuC*Cb2nO4c+}T9{>pk& zyWr(UI0VY;&`1_N-RIG7+~Aq}-&eLyLe2j&?q zOlGAgC4XZjQQcEnaUp^(5+_c8!P3KURr+x*IY8=-vm;RIE(wJV1|*zr;%|p_0zyW^ z-=)ad^`q2I%}~7gVPSK4Un?!Fn2gdHblP z3g&5(Foh_qAj&oNrWFbB_=@c?ZKaWBy2Z(zS}}gPP$^9Ka|5!x1#OrM>J`zZ2yGYR>3gzuTB>EiAFIBPXa1Wn z77E0TnruVBG$cldRoM!S>ia+D_iQ~7bOXx)5V6y2QfHv8fWoQ5SaDH#?icZ<@D99j zQYtFzc!-N5megGG%?B0WBOcLcaRD~8L@kx=D+f-SRgGX{f0lPnQj0?f2#53zA3F4e z5J<-y3r`@EM`;n$XqlK2r$G3mI_0PeXVr!8ZY5X@25pV=T7z2W0&x%CRW;!8Q{ zh)f_lBoRsp5QJ$Op8SPTNY-dz4c@@pFg2W$uwc*-^5xQU9i}jO$bmFCGLzKR?E$WRALE)q?%f2KFjux4k-EfaWZhx>@tq7WIKJ>(Gq+ zNgL4bR=DC?%d`^&WY)_5E#|=&-EoloV}Hl=vIjJOyWSm*5<$wh9=m5W#l`PM2K5?m zQ6xf_6ELor%;R`T>BiP!;_^x*vE+C}L7f zPmpFUGWSJgbkRhuS-qRMC%xv%@amHY{QZ4aA*7lZ!;2*`!H2NKm2=80+b$who23T! zEUs@YfW)8#5^Y-US->MyL=wZj zIFZr~t}KKIipm67sl$uvihRc!f6cm4zYX^4=I_1d{@k{aVd8;iuDx0D9>XD7oOp2Q zsIQ~0Zf~+}(p?KN;BTlqj>4G}V%Gk{gqXiWNGFEI+KhyXT7SY{h*;}X;IM`it(Z}@ zq6p{MFLp*xWjjt7j?~fCE=b#j^_hi&q-(EbmIiEaA_UY^jmwpfOUA$Ys(dhF|0mrJ zUQ-`ov$2j(tLI!)@a}Jh0cV5fx5`9tKQ=Nmb-Gk4WxNg<&Z&cbR9`y!IMm1yz6K-^ zVyl6FkJ2sA5!Ymb{~i7=N$nyw0E>vyg&)8{afs5RP6(7$1w35FbLsybayi=_SY?Q4 zWu6=}aSH1||3JE|g+R`rrZS2(CmZCNNjX%k*=zISw}T*nTIqk&orco(!<82UcMChe zuKABJ0ff3R23JAYWH&57wyH+lNGAjCLo`I;R?61-Mh}-@(kE)0$y%v1<(b4GGe^2i>)ATlHQzx}Td`uEZD9$aNpP6kooo$3*FFyL;Rwqs5@W z*joty` zJ|>TCCTRo*q;7Zl@1iQdF#}PQ9aCP^-|f7$>@u-JiX|ysf}XUR(Jz5aN8R$V@=_li zQLuZS^g)HVv|O1*GB4d=AFz?F;a!G53;T>W z{r!|O-LjxyvX%GlA_*v5?aC(T_YEG990S$@jAN@`<%b%UOw#av8G6BWm;2j0H@wiy zc9Jd{0rkpBVmX@V<~(Tq^6ufmb4~wc(;oTd7}|Apid~YA{&>_Y=JL9RB6n6ssC93P zSz}d^=Ct@xZD(hXp3pMb*0@0BIwp!y>)(3TCR&!Ugag@^r&U*PI9&?T0zjZEG+&h+ zrF8r}2W}5l$VpJVzV$dxqG>&b@G3Zpfg5WVszwr{U0D$AhRQJaYwM0%~!0FFtkDID|V)5tBNRewNDg|S;8() z_rx^{pMqa#I8j%jD@!yGsb`Ft<-|asJ zQ(|>dQWP@ehc8lXJk#^@2P+OmQ<>`{BxdPW0?k-{-bdfily88h?8*?6-2=Vx$t_OZ&<`c!vXmiKG>@> z7QWQ4h_!f|{vg|&ep6syN14A&dV*QO|DG1hj=VONSwyzf1uhqI#>?sOe-cLHL&30= z2SUVvg@+~+z`i_v2G2X+tq#K-vY&cExDxfQ^^h#RQS&$VGLE9_@%@Yu(&a~C4Y@Bb zzwt68$kTp5=j)3OF9G(J%h@MSbP)W|F|yXQ5K@xHi#J{KObtRvY<5#Uagx zl+gyrgW2df&P*It4F+VKe34X%y>W`G%c^F(g|*0)$Yad)IoU7B7y;=*XMgnCt;&Y_ zIcm6p$sZ*SeyF^+KvvO@LbQWE{}2g5Fr~h_!%`icW^eoII=av4RhjHMY--nA(R|l& zSO0HZBz}n~8PUzFw_0_e_7HI&KJelN({f4po*A5Tqjj#%mT4FaqVCD%z!}V(>(V-d zTA#Yam4?wj>0tqDq+xStaWlic=w!Yd=)b_UfAYqf&1?^ zyYedqME!)_{~2cpEA?#mS97H&yMxhcT2pc5qKzhLo)iz_MbJrP>I=UE&Lq*R-0N^7 zZhJVif?}B|KMlV7}r;rr&Ex$XS#Q`!@ac&P7bm8z!GG1I?x7%B8-!h==j& z$vTTE-wCYK51KxuvS;6^*usr;TX`=bdNwnxigk4sbg_$#5=5oXvM+iL9-C>T(7xNK zXa*&-xZK5>kd3&GQ2e;!qFTb4$`2CPA*6$`NzO$5N_`0Ue3@|5e?Og)vsZ#FkYj@S zaP1PqsWC*tB+3}M-&7Xo6-V_iKE;CO#GCL{Yo}hQcwvOGeN10{a_~JB@~<}3bh&=H zSY(AvIw1U$L||ToHnoV&?{KuWe4Y7KXG#~skuK#y)7lS10&_mos{}}M!!V_GyX?nV z4HIDx;#H76nt|jC>E#cX2h9%pTA2^QpD1>;+8A7$O|~~i=&?xUU%wf*qJm5}(qaoo z=*?uh7mQA_yBg)dm{g^wCyx<^eVX)Ez8@wIrb2(>la7~_ z!d4I)X-e-}bZUQz{P=T{(#)C`u(On#Jr^0#ZPg_za43 z^CWgpP#J3qE6%&Ks3wHT%-D@Of}dWVz@t^H#{x%V1H@vjP3qIAKfr!4@QY?c@HNc% z$HQN>;1w1+J9j7K7>MGQ|MielLt^sel46kZ#HsLd&S&Ulc;p4(TwIf}2vwZxaze&J zByDsuXc<4X>MBCf%a!~P{7Wf|MdL7^A>&){w%Da$D8S6-?d!(GyZ2a`OkzIXe0ipc zjxxDqA6}mkebiRmfKDSujJSUis(Qd{VK)CcAfo1{6up?{m;2^aV~6;9?Ngf2EByKR zL#nvJa`I!m&v}@8KVCQ8-fHrb#{}TU9ximm&W`cJ zH}I_srk5NAlohBZ?h`~_#?#*m!Qkh%8LYcRc?0u6ETN-}v=Ik|KN`Kq|8|=kO*)gC zS~I5qyE32w>im2Q;`e=d_E}z<&LBeoa%rfmNwkQ!zv#I&oo~{LBeiNCT=qjqsMvIy zyKFKxaMukUvFe0S$0O^7SH6D%Wp`o144s$@6NRbPH~4~=XS*|IAeogn%Fbf4oH^6V zfi~D6@EYK0VaQ6NNll7HjdT|pyxC;@Tmo54ziVcxv1{|@CO}UW(<*)Q!qHe(s}|BUIE-R zge5Lt5+%ZhHiRf+OAeCk=MsL=^$C7+abCvwyKtK*W~1WdrO427`^VRnQ)_5gru>#T zFuU%un=o-rNOZDAYp!Hp45nOP^#DZ$HMnr!*7!eBDLhHR^1QlJ6g>X4W)M$4m{8qe z+CG)DClTLiK$)6FLf%;v%;M`7hJ7`tfYQb%#>6W~Qbx7#vN_~WyG0%kmN1)$HW)(gY(rj0|LKz7F-o0L3i z{zg&*^rvPiSbAyHyPhXh697BiVHv9ZKOR2Bt2z>Zl=60frTH8b{b_$la@6VzvtY}j zpPyQ1SNim1_=JZzU6+jr!Wp0ylh*381-U}=l(XE5+mH<@V6}qcu8Gm}jW^rmB6E5V zvj56-jr0zY`Szy%MQ&c^eNPbZH)NIG++CNkzW_UFi@LhZkkazzwwK$SPGmKVz6d zV-1~kyuCYmctsG)82vRSr@`G|e;w$xKQg5IfaQj#LTBG}F*J1tpec*CAO^Hm0dbeT zY=;_&0SUC;e&8VQEl~K~45+UbNEu7m=9t74O=7o+l#za>?a)8`argg-IeqnMe=k@N zO{I6Rt%~-xDcLA7X#_A_!4z^2iS&W%yteECH_~3&@A{NJxD)ahQmxnyLG5+ z1-t-|jJB2eFk-?XLhAG$dA-Ag+16)#!zIc~-NOsyy}#Ir&jwXi9xoP-xATRvH_-nq0 zPOSr)p`YHcPe2rp-0WX@JM$NQRe||>C${YD;kNe&r5qz|GN9dMi4iRHwd8oDP}9(g z(je*SOPs*4QQOJXXb+L;jX{T&3{^KUz2w$!)+Z8!PxtxfaT6*Y5dSsWmYxT|BF%K8 zYdnMNThn4)a42wKJI#)nze2&(M!P;J>u))=0Pho#T8+@nME7L%)N5~r@jfO}4Roy+ z-`wpoL4%-iYXGuUD3t^9fv+Fu^h>#8pN1Ak2hyNJVCcMtvVY)TI^&;$*)#oC=;5JD z<~H<3G%^*uwcz**U zJw$j>)hpzqFTmaPDZ&naq_1m1`P#PAafKV#(6RqD!SwAiH-)?`*g2hAcH|S5WZ;ItUA%-m$>)U%x%+!@)euBIJmzQc ztI5oHqC-{>m;iYRyZQ2}K2(Y7M7peb;1kErG>;lkjR@mgms=F)Vc|ddDkDZ=DC{u7 z5(3VssrSU!fKP>S7#8I{x}lwOpwPQ?tqGts`7Gsz3V`Y-Fv$dYS?$zECGygf7B_AS zYQ?j@)9Ybgln%OG=}jN<6!zdE!Q|q|jA@2c^KE z#h+9#*u{UJr9T|NiP}L6{e@2LqMWmMizWHQ>h>#zR9F~3bvdDp2)uPm--zcu$F_5v zD48@^G4p}W#tD^)T1DMt(awK-oxd$L3rSmEG_aN8j*A7*qY$3cgDiIvHPGAixXDuN z>L;V`eLS!GU!qQ<(Z->vySOyLgBOsQ=rF53cj~Zxhx-6v*zHssYo(!*Pj99bW*81j zDg39NnrrS^E}bo@b?>e*+x4JblW@|@Fp(0DxO%h{&6Q6B2yXj?<@ej%yWL2P5SQIK z=_ad&V?r?qRZZh$#o7^TpAPVWvndJSxVH;vH_+>q#E`eVey`zc~0iDKvays+@7) z8BV?mm~h=NieUTqxT>(qIH?JuOvyg09ESJ}05w3$zo0Q6`kdUTc0gj<7tdo4;>)*V z@)j69DD4M)Bs%6l#*CG++jTarLq;~f`+8{EzaPIIoyiF*BINN$wmW`)%pR!J`r|Xf zOTqF^Huiu3J3z$0Hw}v9pv?~VK}XR?03>6FU1Ug_=0rFAX{$5;) z5F4!`(5^~!Vquvey#y7jXb@OXXQIIeHeET7x|#eWT_Uh;#V+tygBB;|Jph%){p%y1 z4@00*p{sAWlb!Jg*{E~Nisw$9=*}wc5u~xmt(}!F>lcGC21^8f$ZoxR(x(ROWy(Wy zlZM*7f{OL5_{=_vHued#9EGYPhC|Y28s(-=s|fTRR4X-HJt!Ipj$je^BKY<-h z=`O!|X(E%FK8$(vqx{UsjxrnKkjN?7De&I-LhhhgPuje!@d>h=lCUB&D)BURMnlud za5rtHb{x>7OxlkPmZ(Ww(OLe##b$KLPIL8ON=)4Rf?2HraV93s9O1|#IXJ!b*JRxd zdE5BkUut;~99_?lQ0_B{oR;eT_6iJHdW0YjyeFH-}pg@2AS7Ohs2 zkReQnX)46ZluzM9VXDK9DBY7cw>9YWI$)}B@XQJE1>r@AJA-e3{=zQ>H|APp>c3)Q zyEM3~ExWr->mUhCNcnA@wa&}1ouXAM?Myti5aenX(RsdQbb_Fp2XLewYyjkQ?z$KI z82er|kWUq)xb_sOh@M`9cY4%pbFt|4x z`FB$}QsBH>tBI!+FOP{sM&Aq}sJi#dWtJohd`i6(<0dad06c)rTZ^>!Eg^5+Jv3zF zD@|vKw*~8*j>skEmc^%gPaWZ`V_i$2xR9igxfq`l-X?i7SIHTPk95B6j^$KI zC7gGR%b35Gr!(XekzVH{)aA9(pJYo55`pLkW=CAk5D@!qy>X8gku5P0>dS0=!d6D) zX?vj`kK)d$3&{H~UOb%`71i=@eW`as3y9?x_5B-n36Wne4&b)I?QS_j#36;DL@=Fp z0oFLtN1wR%!yiQHMtPo3Ae7$j!I0^L$?>l+ByeIOk$GVK=ToYm)M@#!I;st7?{Pg5 z@)GHPZz501*fDx-qIv-8JVFou%W%=ae7nsM68%>j-lg&k)2LYY#(oM4eQG^?1`N!k z1o$-)3(cLE>nau>MQUUz%+Y{ZZsx&jFDI}(E?N%0tq<>#UETR2o^=uNKoOK|xqT$hgofkccvuDtkkqR6RHLbYR_DFQSK0pUZFCGDAbsu`6O8;5a&lM}I_L?~o3wKQ zlQYjbVKqP5ljw!O`;+zB0cxiv4=hTK~4`zu(o$1f}rv$12-( z5vV=EioXRzAZrq{o{$XRMAc2S{Aut49jGoiqg>8e zbMB!M6ps8;D2=SydX*b9(v7^bi1;hUaHRqv38=(06u<|{>^O~>gG>&Pr)C3X;=}_Y z`zg>w2VjFMiyb~wD3U}MPNKs)s_^ac)~y<9bZb^FXb`pqSnr&10Rw*3OxuE+a`5SO zKPB$wc~laFEehP$>Ge5>)MHR^rB<0Bg-it9tr8qs`A!!kg0WTxe08ywg;qxL(1yH_ z&VzK#6FO!jwtvw1O@CP144=D}h}%)|G%5VgX@hBan7mB;b`ws!5Q_^umGGEU@(m9I znJswX+L^VjV-0$w##Syll*tdCa<;gXzhXj;(1#GM29d#uNvm|dKr6DbQ& z4RG){l=!LgY4kBpq8eN4+v$net2RM0i^2WDgNsVjddtH+z(~ksvPljuMv0#}8_LQs zP~Wjs9GIikVmfJI z0dsR4^>pNiV$8-b-4h-jwax>ojRlPDXr}D(;s%C}Py;?aX73r>sbqxDIL_9G$1ORR zO=19*nFjU6{B-Asg*k9gb0>OZU6utq==hs>W5IP}e^!>D_L4sl;=l7oZtl&?bq7K) z1{MleBemteKcvee_GJP<(0*wx(Qc8*=J4m?KxJmC!Td)Lzm8go_ds8F-29P+l0wJa z1GX`u6io(Lj{$`*A_{q$n{h0Z^#r8E+=3A4Q?Gv8Y2U_;?P_NFH-&-WacEW)SJ*eZ z%lG28a7##Znah08*B1dRd-qCLMY~&`(w%CSdpr>4(H_2H8@xO|sz6vFbt~c( zoL%gavbc2NQOLM|FJWDqQ`xvPUK{`^_G(?zqJ8F;ddkmw)y#Jr}={$NRan?nfBg57K;M_cmtfz!KwZLs44 zmGoa_H;{xNn6en4sj^R|w~BRkdB#?CorJ17gZ+k)KZQ{BPu73hX)9dI***TKoU6c5AGt0*gQ!!1#?7zWFOLm|x^!kg$m-SsJ!hLP?Fuz~i{p;s(+(jhN>7aM#(jb!`3LN5T@2v4jL zBX)jGj`PzKm zEVW}OF+7wa+Crn?2%RYWv`M%Y!bLhXjvvAaU3OrRZu!0D5z+Z*KY^e}sDBYSI7-6r_Q<5)4Z6zBuu29c??iBX=%^Vlcu*F zSZ!7h`-CO1RCIUPk&j$4zbixAwjC&#-)iZCQLBMYDraTuTRnRbwhRu!IulIif6O&c zd*s-}ckiZ&s+bZ1E2shT;U_jgL~D*&-Ujb1FA}w=i~U&9k9#Z8TK$Pvb7=&}(zq$V zL9pBd`N*GyklbN2VmoG#i-7=?620*D1vXc)7{h^=bIy+YI08V~|x9 z9Mi##REth}w$5ydFgjPi<{kqip2aR;pqJ)huazdxDYR09CXim~87!5m+SXaxCkpR_ zJeW%tx;zWUl}a=0$-kY&0W2=n0^CFV*z12#G-X zn05bTE=d`@rN1PgI+^x%3e#~bZNx@@tOgE|gGVx+aKL%q1ZRt=nS;@jg}p=`H7Mg{ z`1BxM9v&M|dJ#eG2ngqI7u>*jUkJ03Ittc2^2)4A71@KTG5lG)U*AX z89|apd_E7H&BV#fp%N;>{<0Mh{}1^^esVh5w$Hh#J+&?7R3^h_jaixPE%2~7RhSHS zWW~Ji`bvo)4pgTR;wEMD`YY}S+&wL8<;C%(hdJoloAj5*S*b`JxNeG z2BXt|q@VIpQ7Z>r1K*lHIRfNxY&Otyk#+;gro=xou`$*ewazetsc?6n28MGl$WY!* z{m2HEu6%+?ZzCDhSGTMA<11;J_)Q0hhy>4jK|4KDGC+g2p2Lirkj0Df$Ih4Sht^Y| zOnc9OE4#gn$m_P=WVc@fYKsL(^SXqo0Zwq7KQq$5z3XB?1!-9(LaY$MehFQW)GeI+ zZJ@yGL4ouaELyJ}po9wlX?J<975UMCpLgc~F{c~5MQEeoFk1S5*;u&Txr%loec}BP zpr#Yx4d0gP^we_$RTY0+n3>3XvBz&L}Cf7{x6 z%DEaw_~2{$Furinq{l=Y%^jNlwwRK1|L>(033)-_Yp!3}HyB<7L_Zi=Z14QN^Qx?x z{Hta-s^*t6MQ%Hl;s1kamp^Ss7OzNII`6B;r&{?NDeQtKD z?i`)yR{ExVVkuhK!H_liP91YD%euV-#Yp?&B&U`T!M4lN{hswWp`H6f!p4~_C{JM!{yWi6l8$nXG0D6*BsO>HJNiPa#8_i?nKsd<4HBDuaW5SWx zcy~b0G*t{%04}qrESX=8^v%_@KScLH`4~Gzuzfv0)PX!0Rr+81b!z73w3%beh8JCp zugYw3wD!+2?hw%ls>Br1K)h>xk(J{Yz?mraZBr_I1A=OB#oi&gTy9M#+{026Xd={D zGxfba_l{?Pi(}`aYqVB6tHv=}Dq$>J)KR8C4v^+j{~ksV){a=`b&>v7RI{uDUkkIV zs(+**jDml{WD5~(I)7rd+&VaYr?75~G0%MkvPic}x1Hg)+e!e#pwswgHE{+`Elt_y z^;;ufu-%1OR;)GLZPOZ~pXbdOm6LuzqkvW*Ub^D*xd)9rcvt1x^^ZktL)Q&;b$zV) zS&-c$xUz$in&G5rY2ANR-4qAtuh3E-iLDqA^owf^VUBS?NlxnI_rvb8(3V=P<3l}c zw|Zc>@^o2DqMN)GBcX*w@;azx%Gee=K;0v^Z=Jo^7r+wNjHao)>!*}oy zdvR{I2mhY&`o!YpN`w5R@3V-`awiKQ!+hZdPc+p8`4vV#R3KX(5Bq~eex}m#zmln# zAqbzniYY4;{w^+miVdO@OAWzpOHHcd7Fr9M^2GMi0Y|>8W%&ao(2vI%R*SE;qS|F( zV!?L+sqYv!=0md*-w9w7xPmyS^^XTkECWGdLj+`N~K!yU<_l=^s zwEdO@^U{?QMY9D5b#wLy`8XfNn{?yUUcbMPiHdoP%l<561XPac*w2UQjsyCCnSR4` z2cY4e4OKVxs}R6?-bo1(JU_zDL9VV&*+J z3psy)06}O)&?r}t=*kE3X7|z5V)t-SsLPUQ7#T-(&Q`}QUZnt~45!kOCpHlGC`NPU zchJC8>oW`T?yBZ|p~8=f7W{CbumdSy5*=&rv`c=BG(*@6rFF=FlzzdEYD} z5?S;DCbWNOAzg)g+*{A3xXrX!ml-ZzlTnZYDu}CD$7>%sW26-k)E?@k@eB%Kcl9au z9XL8Z6Rn;H_`fvnarZOk7of`gnX`a?!bfOAfCh{r{gM8F7cTU@jTc^RAg`zg){3=` z=OfDW(gBF~NDx|(&t1B~)UtgtSjR*bE^`Z|+$nmOcfkv@B<6a~mHBQh_$nX*r;>qa9YIC1qOl%EwPb0WlNX;cR!*iK9rZ$ke?O@!(I_5B01e@(dl?!n-u#+ zn+40Yz%t)N;?Bwh&m!*bC&#};4&_ii>uk%5II@rs{Q{GLh5C)qWI22~6^uign&Y8R zyG`FVEM*(ITC{;%_AuxhM#;xVLAB%zB!R!?9MZOLUZAt(01^b^P-KrEt4b6Gnt6eA z!kFvmh@@qDBiAid>kTtjQZ7lZ?4btvC@1G+F>GXajQg`79ICu;u!D6?1d=ySB`MIR z{;F_N^3Xq{8lP(*C`e(G84@SuBq}p_e{LpuH+d*4pf|1_0>`JvP=FXdNFEPevzU&u6(LSc1+ z(b#ta$ZBAECgciNri8z;P23Vf;}L9~;J2rYjNj_Vf7gLSCw-1Wr5gihaN(ZbH`mRyjV$ZG(ii$Qz{?rytkzgzG{N2bY)Y>&EMCz19H;0(oR~hF@by2FO zDhi5Q(swky&4&)qXP^=kjDgBQ#&tW%*OIz|ouXylKbP?uUgWKu%aBo9 zDNyQ98qi+)o3vv6kvjKMfvw#oF{$aZ6-?ARfQSV65SVnlZ{>vH$Z;~wsqBLI9uYGI zG;+jn1AJH?#GyyH`OT@s>HXdAByA_rR-f-GofI$k^5dY?ft~z`lNU->0so;w(tMY> z_F1*AjSwl)l3t)QT|ek9bm%Qbk}{aaS)yCk7g4ntL$6udO37^m2s3d(=XBI3yW@)A zljdpi)+E0eOn0M1my4QJ_?du$;=kF^9Eh+V9p45sO*+|2xu1`%7*R|;d{Go6Mr3to z!f-Fv&WvzxFOvfjY=K-ZqImyKVWX*Ig|sVX+BRi7@(8|zxY?bMbL>`L#o{y4-J^u> zrQo`EI4W@1Qh)Mn)MqQ*M_NEz&3P~s2p>A*v$W>%XFEYc9=!MXniwPC_lAslRr#@7 z?q$}%+D&3%@#Ql}v)IQ*F+7WvJ$`)``z)bY$tbO<`89j)t(AMBtD(9O-#I)OzkP`` z{-l7V++0?~LT7R}oB=NcZZ@wuMACC$x1*e9{h|L1)EaeGCL$^;d|$Q|k#1584Y0Do zbD{01!kCh4;2#2Ftyc>|2XCjqFL&EI;n7d5e<(=U#uk-6y-pTMJo-9ncECoaZO98cH+&O~X#&uprJJ}2IJtO_C z9ON>}y&j_a{TNrI{h9NyN*8hiEpZ+1 zxLpXM{8JhuCLiBo+gAl@lF`05>Yh&b5}vu$eL_AjNit)oRm}n@(#tXXh@9=ai7+m@ zICe$)x=4p%lr=;1V`tDY7NP8-K8ezX(`Ch}Su`PhBz%3*s3eO&K%?appMrt@(bg7| zd}aFhkc-ZVO>O`Ss5jsL-sWN1i!Qq(6$q^Pq}7NPrAU-y_K%9 z)_7#B9Qz{$+wwA9&NS19dR?H0SZ#bvm||K1C5~d$l~wAu_ii&0{OXk(RoP&z(yG8g zaa(Be!29&j%T(+C)2_n|b69Ccloy^iWmh*D4Zs5ML!2FWWp7mTTU>bDklW`ljEWgr z+?1%bvgXKqfbJbSC@8t@t*9|sVMXgL*=oVdW&X=DOb|xk*M#*(Ao?sd3>Ue(dqIPh zRQ!h(u`**Dc@TmxJ@&3h>IO|F+KPSS_OR1ydkr_-7PR}|GT=q7g%>TapC%KP`Q$@& zmuCK7B4XhkaIE@^X3IJv*8~=3IGQr111n7m8S~WHav9rtoffIy=bX=5M1LO_zwfI? zF(K92JP%rYnoF79Qm-V$Z~2LN$J|i1c==qL(aW82#kM#U=D=>u|5@%Ot1z3ZxYw>r!C=yGD4Q`TO&^)yRK&I8x7Gx z%CEC*#?^&zQ`{C+8&~>GGIVC-!B4-;-ZH5&Hn`_KwHbb5<$RV=a#sE+i|ecbKKSp- ztBk0xhUvT0x!TmIQF)lQI@w$1%T{lu>Y}aL!f`><0X=tn(ENr?ejhVP=RO>bNOWR= zuzEn$t9# zLP^i)ms6ky&5b6Q3ANk_+o7jTB&z_f(=wC+rtTClSj29ervem<#5AkRbCk6 zfLA5Jz*0B^Oq^r`ShN#Z=c7E3vo_t#m%?-iC z;A+{&)a5LW0{walkNYVIchBtRiw9M7L@T!zw3_M9>(ri8$BMwEj?FmsV*vbZsx0BE-Ng<~D%@C?0Q`Ld2bw*WyV~t?HzAU50KE{ABUi4jm}Ac=RsqNdIQF1!WrNei~=>(Asj! zcXSGl@n9Zw;HVbseBuN`xtaeyw9vx(_ALwzyYBS%Nv)@CSUP^H)Rr(|q0nKED_sQP zJ|NoTClfK)QC3Q7Kz)N@={cK)AT9@N0(vflF&{9YY;0CA3bP_kg(b7ViIk!k5EtV$0jo{g;H0a=Mg9d|p|##ZeZ}yfqUbTeN$2)gRs4|wvsz*Bg_AJd zr+$+UEK;)lyZiS>dGb;O5$4YD$0kqvp?kV4L>rW&Z?$1z^HRCL+y&8>QZn>eGbF6{$FnnsJe zN0S`UYUfdQ*)yuK^$yul0hzWyj=1W{>Rp0{6z$a~w@^+q_u-US^ea;yZsF`A>s zDb2>Egsze_v^Fm6<=G;G$Yv61<(6g!gZ39s*Ygt%hB8qVU6zEiL@`7;=E^ zBi?T|@u8+45x6g=r;o9$M?SfT;yn-Vd? zXW%51+sKVn6L??n5^Y4H?|n>{$yl^t)8>ahUxEU?cPlWb4nqx+9V5*Dt?Ui`;>k23 z4@_2KauM*u9u)$xTk;d_&->PqYAT}kUKlpCFMB}aS}-G@HsNDcSzd8inVC|j^|%zK zr9aNlbfrST#2GeD3*~MsSXz-JwU%!_5!1y%AOxCVTYjz@%Ki8o7;3OG6fD$w+7kkM z!B3GN`r^1aX_&>USzAits7-%rNUacL%cAa#o^8#*)j7cMod(UE0w60y)lRuNk%~k3 z_eFYmT%o+_N-`9wE)h6$Cr3T_MhA^+)n7X?lLvMIH>V-OS>d(QaZkGwasWp%{uHk~ zo{0WbwrKPT|8I_=L5RVckPx0uDuabxvKn0dumrohK`ezevJGw$=2i8>e2;%l)T~E1 z%|8j6XIo5tf8yZ0OSK`FAurtyl9;8=s8nHfP*JWok{y$0(^sn8>T4hZ?57p5c$s?; zT+(D9>P`0g;2TJNVO%L(;kXz@bp5a{ve@}99}Xy`XV{8LcKS9O(X_}E6EXcl57Lhs zw$Fuul=J;I29qSL2{lCQ;Egb7l^}5vMqj(edW{EJO=aL-qua#R$c|kEFih3em)(&w z@bdg!DLWwQ&`Ov%{zMUtqqIg3sf4Xy7P1T%<=Jm$5SQ~gDzCn)?DLkD*zW!qbBt?` z8N*FSI;)_6_524Lopl9^(!ng|HXoypk+8oP0g_}=JMh~4*uM9GyoJSiH3ax?OeUJ) zXzyC1jy#`6^=soAlKX#Z1NUGv;H$0nMDlz37Oj@6pXYH}RD8dh8n(l*GUOxIN~moLd5-Svf*d(RVIdM8yfdHixV zBqnL(v?w)&!{GoS2CBtLg3xS4L-)S0b|i-)MO79vD-01W?kg3u&`6)%nV*b01G~1+ z&Hm*l5aOB=TULjzk+YF4i_=`lYCx63snlsGU8!D~`za=Uh70^o@c>g}qT226- zZt4f9!9C^t-;mj$I)yRQwK@lMUYg!!(;|hi>1)q%rJ_#az~i#wYOK$vWf5I83mS=P zAnN!r9Y@Z%RL$5-)H<^P3GVOIZ2F|{4vi7mLG?=)nprpWCfZ_+rZlE(hZc@?x}#bm z6^{7(9h?Sm9>z=gup(}HtQ0Sm5fQRk2apy#0}^_AIgk6ZPtJ`;M^dsfd=|U;+U9Rc z)q3C#{Cpx6B-kutRMTU)D%ack*BQGL2N*;QoL|4DIz?^X=b}M{GA;#pag9I4kx|Vm zNUULP+MADTv*kn>mFd`wfQ{YpvFwnij$0o6CD|@dP2D4V=nmu#GyZ_)-!m2dT zL9V8*LMiEe9OW^$Cz+j#k^$tsxicWIb9cQ`}4)FgQ8(?F(p7WqtOPjGF@&e3{RK}uA zEe&2gyQYbxFOnrb$Ovw}z4|5hzg(%GW(9}&ox)8#l{i_+hx@I&2&=bG1y^R3^N#@4 zIfml2rZK>!fIS{}gF(>y>+*tNjxAi#pyzQ21k|G0yP1oU1Qf$_-E3)ENZ&wu$g$-N zY$GE-2on0;m>Qd~s?;n9FM!Q5%s0meex5i;*%gWymTtOL+B@%Z{E!~9nv0saIst=g zB(iAQZ-;x3T@u2MKF-q%}L1_bw7{dprJ;L*T9~83e_ zYN+{>j!yWmERzB59Fm>6HLY8>@q2+Lm=D&PZ#3{)IG}wSG46Xi+px}=qQlU9^=O1v ztWhIuQ1y+Y5`79WxKJ%t+>xGC3#-JouHUx#z<7@D5>=;^1$414U|pLPj}(bf{vvzH9qitF zzV*I9a&>R9GH?*Yh^*W`iztLA44)BezSPYAi)-$L;KOa4V!cd^v(V3BUbvw@2}(hf z9cKf(_&0{{?7>gkoM_;kaoF$t^4Lmfz+mJJ>vx86yd}-)Onye^sm;xgudJF^g9}a9 z(wSR?zKWN+K6ivVN#N1h{p_fS6g!vp!_Su8Gh1YojCW~Ex@5*Z5iMIMy+YJgAroT5 zd}AB5;}!s#8)P^ZSB4lUigNAZj_mQ5b^qiGO6McN#nporu{E@p(L@pOAQSce!!RG2z#TBNgjmj;`!%50w=00Z#@!7McxCF0(XJ$rYtCP1La)Hj)iD^f?EdXjp>Jcb21g3Y5OTB84jzk(kqFPsUn`PbP_?-x}y0>c|F0X zMKXP3AvafA9$##u-g5P?j4jrz(ZTUJ(vctTMW{+lP5C zaEK;rY?ZjhRftKN%N$ATh}z;zD8mLPSj!hg4aip+5HvQHsg-ZNzfo) z*J!Ri)uM^n)QkG2C)A|ZyLG(G%Fmgu91cN)7^Xse@`!8sDcxl$ib9V0@!&EYIZ)i0D zCn!taPVd*0#BuPm>3$ckUh!o?l0d5>ky>K-2P%bz+&Gi9P3QpLCN-q>en50-T&49sn>snd^IubaCc_hkXA zbYVzR&XeFXU*Ecuqhv(bg?_ui27p_AIY)&5iw9BI?W?-Y^ zSTfDc@X0jgtp3Im@iYmcKC0l#a{y-GC8(FAuk%RAVhp_yH)iuAXcO+tw37S8PvhlH zf&XB1bF0@J&6R8I?*<~x_fpQ^6M|p7(VC+Sm1YjpbMRRo|Ic$4H|lB6i7M0Um0R(uMrS$C49mz4d%cb&B8Jy|{L4<;6IL>3j1C&$W!jrig z%q$yqL{+}HX!TSr&l;>ISF`vhGcjoKlgH^Pp!&+Ra#hD}&y2+j;`oevJU!QXT{khb?sKaLg~IQQ#sd0<97eKIj`3l)pUh?%$sZq5a(~IJ#lbyR z#$jYe&ab1bVHp+2Sk|uN3ncc)SW>MGQJF*j>vCICm1Y+6GlLBfvF$c;SU140cuJ$0 z$hUGaTlX$Ju~czAm?q`;M4|pzMD{|eZM+DUN^wIc6CMz4A>_+cnqG+owLY{fJfKyN z#?fNmswZW!J7Zob

0hg`=ORUFS&j6}jE2^5Vz**xk_zhH=`S^K82XeNMe|tLJ5c zi}Qmr<^dSdmYD*b6;(wQ0!_*jHHrYW8a#ILe>+9v3{sQuSSo7~F$UpOU{*7LV?+JH z0eNCNC4=8x>~fVbNN4ane=oMW+wlHT0O?N0P~6B($NBHLYmyZ07X2#wdHmQk8N@KD z#i_W^(j?cePftD8XP+xGi902--%73fLUaZ9v zCcns)4w>+gK`0%*Zm{@hj@lEN>q=b8Bb%m+Dw4B=Q zg=x*(8l8%WecJ&68S>8k?68i#ShHYwjF5>>q9jr3pNCt*Pkn;AXpGP>R$EojZ*m(q z(4cdObH2tjs8-Epj-qrgYJ^1;(YW)sQ&r7zj;ZJz$QO$D=4zCW^vQQ$ss#X|E$mF2 z>jZmOJM%K~_sq(xGw-f&0W9QpRhi6jIC6TiS~I>->DTA0n7+TD5x}qj=7zA1r>y7) z!v7{w0Gg#*X-9AKc_bI?lBQ1Z96xy8v(kA3-3V#;Y}tW=DGrJs{Y6MdMaNji=J-=k z@@3H3t~J5T%Fw6mh%A5?sD2qb++|=RP4@P8Bj(V5aLVf;6O_Jj_%$hgeYig&aX~8jibS-)ym%vsCGbRopsg5)Hz8P6_Mejn-&edm;^k8ok)cwyC<`1dKK4WEYt=!%Wb_ z*D>3*Bgd+7*?+;Ri3M!|&EB-alv0A8kPPtteXfRi0*~MxM~ih6wKv~7oBcbmW;%aF z%)t;M**dHI<-n4R|00v*_>2l7oWxlM_E**2mT_0w0 z#!?HlJ$5Q07$*65!pad%Pe&iL&@>qpR!vb~e8X4!ABQB@CoE)$F@hTa9YHLMVQvu` zIdKquNr5*}09 znJxc$u86uhaY9Xlyr*!NqY(daXE^^}y;xSAfhmRg8(vRHKf;LC0=YA~bjNLwQm9w9T%pAqe_$y?#!fiik8W0dg&Q z4*NKv(@=4Db#Xa(Lu9qI9XtgA`_SNB81#Qyos;C?cvMqG zOh&ktYdggM`{fGH;`xAtE73yTprNB^zL$Dmp+Y1$4aw8!HtAp)~+63L&ZeC$|&qK+z-KEz6Xu=d8wZe7Ga-Cdg`Y_CfY8&BMq%*4H+NX+b>joqEq5I zeU394GqsWttQsEz#;fa1NWY08yxvKCf}3Dfd40i&dO~zE3{-#{_Zl0gKPZ)SZ;|W2 z@`*=z-8eD(qkVzjMv1TEd3@~c=rn> z{mo9S7N!_ZO&^b<0}4<|{pN+*HkMO^;!;w=DS4}F8oX0pi;D2weO3fHhyo;P+&Svv zw9F++Of8AV-ffNt{=19a_l?#InWU@?Wh&vYw{LHrt&Vd>($UkgQ%L%AF`GeuaP$!2 z#o%NZezynQ-12iv{9d!GEnILs1;5#oND1lc1)k8Twv=JQ#6gU)73BKcPq^nJULxV)_$<&?(JrK#8oRg~Dz><+`9le0^wMG}OQTJ}J z(R$!Rl^W?pVR5al0L zYCxCrwq59V)2w{uxB1oE+%P%?Z8bMD`)~UasjXN0OMe_wUvbItrVnftk>*asa+>VH z`<{$XY8&?RFx^XFmk=m$T4GsB<5?C6-$=SZSijx*LeG0aL4Ut740+lvmH#WD6Wqv9 zhAO`fap`#*-VRi#Ui{W1;LK!$i+ruC)S*XmbW5-k#Q`nW`VTY6wEWc#;TiZQ+Dv8s zF26LR*BC4U}pc+F)>UhB+dcngGIf-dA=udTFdl=Tu&B>LkE*+2X;WqU|gKZDE=NCidl`8uXqNa zu|k5?q7t6oDWwc>2w=h^)nEd)klDmLukV7+q`0^g<^ij1{ZmEEfGg(4p*~jpYUJh= zR>=fDB_Fd>%|A>Rnj7YxRXCyW{bb}M$!Ay@jMlf-0DPix<6f1!1e)Z9{CM3ioH5eT zldMLp1Yj2m%a*6WlP+E9EmVY=vV7Gnb>0B%sht*DC%MC`&SxLk^8>WG4JJF@QEQDK zHrR!+m^{l=KcB)qxNPai)+@LJkxQ-$!L_x-#h|zw7yJF1Fxez^t=iJ&$}pW~G|JFY zt+ZqqBLa5KfheCfi9X}y%bp0sL$Kh9*@SiOsHH>DqStJ}Eh-7#f7ns^n;#a@2e0gU z^vRFfE$WAOP^O-RxyG}M#$oceYq9ivx#d4g{Sj5v$S||3*;n}i!6It`GICEVV{PX2 zL4;&L1t^8{$=r=cfj}Jo= zw+1?r$*dA9I$~E=VbnolEjw3VK1+E;unXW4G(8b^U6OHGG>_VZk>kEFk*8EIf>QlF zV3T53kw71q(2$G-R{v{S@1zJ-3EOIG60#)Ib_4Mr!L*P>Gs+;6x5sW!W_$@Iz_SwP zbSK3NZ{&r+ub<9*8E7i3PVf+&k4l($>rzVjCN3(p?ZUU2T)%yRa+rc^nzF<2#Wm(4 z%osM=KvP6ZJh!>wg9wXZ@~PLzSyK2?*Ad z)p*^n${A05Xz+Gq12Y!AU^@(|NWS zv{8sJg5Y4d&&>*c4$7yB?+*h{#N&Ong?}|M07XE$zm5S&$CbF^h8eJLU&_uMMw=I; zl$g+ZFF*ZtG%_lPHTQ4Cfp1!mUW?2o~kX3US{sDZDfdZ+j}15)B=^ zn#2`f>nOa1D=%6=1)<55Ec$}(nnM{hen2YheRFM|0W&eg?fvtF0?$t*mi!@5Iu|r) z?fGZO(tas?vU$dCcm|i}o$NsVhd^ zVsOWX1&_5+n1obu(FN1YkB^>~IYtZHVa631hreIAtnlFlp7L$XIhI^8Y6Sl3-T^T2 z?q5_f^H;)7k=cT;X5Jp?-3L0>L)3OWY?l+SiF64g5?N69;SCC)bFIh~w`6Xjw@z|e zBFhONPjx2akbjL=WIM8(*iHU3EF)$_?BGz61e7zt(?8x;xs@i#<$qaVQX4H??8Kme zbQszeDF8iexuqh*1g6Yz*$B`dU$*Ms^s7mJI@=7k$Q0oNF>68Jz6uq;Wo@G*F&8-l-Sjyq>TlMH@qX~75VP^`_QUJHVAkji#&0_X2zztUy-`p&2`eZY0;bf(0Rj+&rA853eFNB-Nm{vodON2Rwn+CA0mdQ`CD{ z(1T*|LQaM*MsnY4omUGRuDclsPz=}~6Q4JXfs#wQ|&Ua`+8O%E?YderUdeIyqujS4>*rZvzt-%RjFwWCNcg_~(G&!wahKSYT z?t^ArrYL;uz(B&_?oP8z*Ymy$GgUu=$5}wMn^E0e?CMesVq^1RYbu}PCa~?*S$?_( z>vbu;?826iiTe(?xjd*%O*N{dF^SHxR%o4>nzY0QGHdqvUuCC25v}@~-Ai?*7V^9> zt)K_;?ovV9O7NDcgep1OMU=`t1U%U4;Jw*~y3WOWU9b>WTB2uoOlq56>MT9d1_R%# zZ<(l8w@;`;l7QM6k4cUG;s|QRX_4DQhF}+DDskoRCwL-mylWf_o=ytclV!>OwKpA_ z5YiE3rCG@BYgT22Yc4$q2BrKQKu8v2>4War>*PfzKm3O3F?&W^_qqzd#X#0CEqGcw zWfS%SWsZ*teW!ML5SRS(juV3R*wC1^EHlO=a!?XH%eghV(fWXAaaw@oICVZZ^iw+Q zM&E$DK@sEXr40k!l<<7nR%QVgu*T9kTu_`-CVm?UXJdGuESwch-pM6y&HboUFRX?BmFNMjpQi z&xh=YyxR2k4+C95gM-aQMJC79x7SwLF^$?M-yHew$C3TuBQF<@x6Exi%j#-1%pb~K zjPcNvB^t^qZ3zTA&xJo@np{O+gwqopkeNUN(3&EkUbJhIc*@w zB+u+J(mNp~&{mkaVG{#R9hsHN2zJF4-ztzp$1Hl%dI-B5HFA9Lrx}LvA9rEnr1Pyv zSmEi0FAc|FAOIu)3Z6)Yi6#0%&72Mk)_7@*;v415N0&=!XT2^Yu+oWTXwV1RS4zqA zQ{=EO23wo^gPsv$CwLQ{MOn#Rz)?DerpG$n_X{GQnf%^KpS_k8kanvoq4 zvdVZL|8;&x{;M6m@KK~qJ{Hao}GbGa~7o_}1$s_aD+%S2*HO!y8I9}2U*?m%&N3qPMlKP|^X<@H4jkY~5h|vtT zzl{avD;0_xr&mkKl-Ji|?DMsw`NBGk-+irla4Yb=e9LInI)0h*(kO1r*~ekIZKjVW zdWb`tXKSGeblv=I+Eh8FM^me^$-!v+y`*ITBG(%;Wkk|awSv=Om4UYsTAM@%;=gB~bL9#;I&ZGryZpp}t_?7m0|# zUDeD?^-BZ|+jaxfkWl#mEDSla519pmW+1r=Yf5QE4K%m1Sl<8=_E_c1W=#J%mkb}G| zhk^--I^BwVb{~%l3av02Nfb26`uSIdF!4YTZ!t%lZmcCQAHeO{RVR#)l5T+*hRrJ5 zptUAlM%2FiS#dS+`ARmgFe@fDQ~6sq$s>MIcVcgC*rXWJD_ix zP%Bp7(^^BJ%#qsG^kJf~Wh3p_e|1m&eP$ zKaiOmdEl=m4_}2)!aY;)AR!&1L{Txtzl>$)@KGpn7jxrNCSqhAL4uHu{QiWVIKfgj z*)Dm7GS|Wlv>G@0V-|Az>vVS#zsm(9-{R<6yj%UX0@%vm3X?`*(s;YAr>`gLK?xi? zNQvy8h`KS4N)~<6u7srd*ZnnB)3pKkb0yT&@nkZ^YCBkE(0~p-t-hCYAzLt@?RXByaq8GRF^{|!=EwgFJM#wm5KjNB=M8E zl++k8#jD2#|E>=gQpSXv9NO;XK+HjQb&sd;h*Wr3c2;nNo3N&saH3l4^=s0aO)QL; zc-U=2iJ<+Sy~U}|JbiCzpaD$+x`{2ml{yX)FnjgRGuzTEwofQKE~AIHE>{j|l0D4k z6)V=RIP3nP2%&hbJktGTRb_sJLp~$W5bZiZJpXJrMvHfG30#P~2D@(1TgwBY`*uPp z=dJkzy>osqAEwzdGx3Hx@4ec;`@0>PE}v~eEmHnI1_ElzUPh?+YB;LuX4#THSlE{% zruTtuiIOUy)BkcL!!FA4m+Dw$DVqF6<|m}>M6vR#{^RdahyGNQ?4 zv_9G43sFo2v4)T2%*h*^=-3J4uLc7Ma@%_J0@8I69t1EhpXc=T-&I@;;DQ9jI&wzE zQ-^MB=Fs!>NK#L_RD3<&Sq}~Wmm;w44w$YlhoR^c9#{YukKGni(dhswOtI41mC;L{6$ z4NHvekyjm$PXX~C4hbI!Ol2-`K0)us#+%^)UC5O;%j1fH)3(|^Mt2fqBu5}M^Sp*)vGoIcEH`f>hDx{$&XjTG8IsxNIv1WF3_ zbbE{I`_#q-fbBIf<9Hl4k66?|^>R*cN~kaNIHdL%l8eKR{!bEa`nbN7TSeQ3F&`n6 zXSZ}FX9$#B%-^{2es7}FP)5=dRx5B_e98kYLz!!@%O8xE;16_H-~KncXMs0jpZzLl zq^620%588FgL#4)CAE%+&7j9DKTeM;25%SqeFOtx#T|O-mD3NzWpKG~HPKcg-$}hk z7$n!J!Q+2d8lANbn0;;r@?=gR>*WQ7Ib80DGsV?rAK#Uj6|z;N5>U0uR`N@Dk22|B^PV^{ zS9HC-jEVXsyYp&DQBkKy$+}MLUDiicb{%_J@`w#}MOmb4-)yB#Ur+}IwzySXFM;d4 zY0{KLXaGZ!N&D$_oP4I5bcosU*Dm;Y>bkG{jZ%eRaR^B0SGFP=`V~(v*`i}mxVTY! zeo(*A5pJ?r>oW(~RUncF9AqBGr^#{A{4{nMouI~fF;gJ>_u&RB#y5tjgX_N5g?R`8 zCYq>jy_tRpJHdiYCto3N=n6fy^^vV*ctlq_Dsb*<pX%S%D9Xx}!cgFDfLmo}j5S%8lhB@@ z!m5LF)YLTBVDDGi3=qwb?%n#53f040BX(ZFokwjNSk}iWTp@)LK~$)QVxV~Kx0EDO zUaK@+d6iubt*Y2>%8}J$ff${IUo4)HE%{~2q0+|XHL}3#Y&u#?8W&QBdc*OfA}Py7 zr3f&UEt|O;F(3jpJ~vw6Ge(0@qCBAlSNnFIAK9}Kz4WUn7WoJ<;C2=&RPQ-+GxAel z(+@_kVA1J<(e@Won5o_+B+R}`(|=Mwz2B(N?8XRv=%wv18&2|j<;~^WKU$)W5(PPV zX6{rD!2#V*AJ@XDTg=q&L$&~DW^fz;)yE-od2mKtB?E`q-T?|slbnMYu+%E=PSK7Q zPcK%>zNlcMy23gfMR{na!YaF+8bq+ROy2<`D|Vew|+IML`bcRxAHd-)sQCnnq3U{=B% zDt>mC@mS9yn~?!ZnhyOBYMLW;l7Z? z>kMp4T&Rfo>-(&VAh7lU4xsq(py&0*<|;J7z9%J|irx%GxoC$o+TaV#==8SFWJhAO zM+s3*+o$!g(~uTRB<@vjlR!1T#t@{=f^U%c8Hf0s($eh`pVI959?lAAc41VY_d!#Z zy!-SP$!*Gfv$$n{jQDm?RYa9OmjoZvhhh?{a;6g)1YZ*`+c&e6&*XP-0qK2n9$Ug! z4!lUSS1Z5qd!!lGGK^#II637FX$4m8ZNaA#HHc;Rqu9fbJoG9cYcFVu--OtuFt8J< z8;kpx#PG|{JGS7J{%+K{PlOIrzROtZ#bJTXU+vz&C-A4Xs{PrL4-3hVdyXYH_QFjM zE!FL(Yl|~W=C4h%{6bg!8+-FlFSW$$uIbBWqxKH3qR$a8t@!t7G$3QRq}*8SiIMW# z{qxlUsq%FX{1xCtq=07}Zq${muu$$Mp8(BfK5C#(bVvl`D!MDURd-kMcUV>VG(}IS zYv?vIW*17s%Sz}rR(e_PMEXtBkIsxj5W+WdUnhWA+8AOPH&`mz;DY^yMyLo~g{lIE z46{5$CyW#kFZ};;%ck=(WpAu>OTfj?)HI&HBg4L#{yuQWkQ8Syk6Kgu_w&O(BqR0M zS&!C@?hwzlsZM)Xk}Dh2{bFZls>+g=H)&|k)It8Q5Oqqw7RuuS{G(-p2Czn@-~z}A zmd;HaiAqnuEAgYa7E*Fm3#sH z&QN?fdbLM=xrb`e|HWmTI< z77*}L@ztw0yFp(SvfCWeFkvNq)wzWv_mxx*%|XRXAG53G%RgK#(IlT3s=6EF1)Q4C zPCthZ0eDzyB=HR%4VO-Bdp*e_#wZ1Y^-leuKwO0)jlq=%Ct>q)KS)_b=~egAKv&z#FK!3 zb88?Vo(Ivg9bHMqVS{=RZT6e6jLq~-jjF($&p{4Lt{8AZ!PS_sR7@O|We=2W9;5>= zXdwJPw$z(&4>Xl>%&RoEg)wu1GsXdxTx(89$Wb2s>(Jipj|gzOz3;*J|HeBau{&M zx@kt#iZp$Q^G8ujoSuA-a>4s0Wcqj?0!B)#5D&6}F_%7dtT2{;T9YZ->Y;P=B>ogn zJTuZsozo`ZKmex46oSdBi04tkhh#vKL`}3&K_irLZPvO%H_(WTw4AvfvWD8M>3AS5 zu1eL|(h%z4!&^viqtU1eQ3Ed#ngJP=7Z2~> zQ$FIwjAJHl&Zd%F6zyI9?{lF&GddgsghQLD4qGRPe^3@}mm%ObRj)t-v`MY1=>pPs zPP_O)Z)nx~!TL`~#08)?F~Xjs`>Tvlb6e!u{g%$^xo59 zyz2ksu(XW9{~ziuD1m%lPR7;c%L0CY6MnqutAYi)s*oaNYT_cq-8F_ep=N`M>vz;O zk?u&EH@2ey?gF51nlb0zkblgvei7~!Uk9H&)klj_GZXJ}F}BifFnw7pGtg{B z17f4F#y(3_+MvRVn2u+Xg!Kiojq$=H`U7dNP1^FMOUI#aU|Xc-Q=pG9CU1a@pzBQ1 zSMmK<92}E4ENqn?6YVon*gW50H^CbB_^yd-5VWH0u)m@W&b7{sbQz{C(Qfna5@wgU z2-NA@(#+uSuO2X?J)!->|9SjEUgoqo+$Bp=rveKVfY8IoJ|qe5GzV`r346kZ^+ucs9^Uk4jCVr)8*m>n2Fw`N;>jPq@+h&DhqoJIju4|B;z_cyS zSyCyJDnU1nplg}6RMZ7Tce1*l_wSQmN-JsgDIi&DM@2f805%~9w%#|@Q~;yU#kWsi z?mC-S(f(Yp7htf_4*NZizZmoLPg8St!TKjR0OL7#WV}3c>lSOB^BCA}S12y5DZ z%L;kT>O=ExoKqVa3>15ZJFpi<$}}M?Pgv$nLBQ#z!Tq8iZD7XW5*TNn9Ip#+kJYxJ$kdhrI-TD*!^8M{aSUAsPaY%Tz>y$(R>G1|`Q&0i(x( z@aMrV{X+}41YIMqmkuc#(&+m)V8m0d28j(ZxEtM|zwC^lz;Uhq$n(aeW5|ev4Z^Zk zN@|UG0FpN?3ZyT%=a32?o!sh-m>==?{X5#AeyU{WdjdTLEZZeGBp0f@%$3yt;bHkU zK1XzRN>u{ifyCBdk3Xj=Ks4yd=&WRW2U8|Rb!mwziVs639Wvp?1isO)MG2ZB;Y?v| zCWt z|2zYk47?0`hlZ;51LBm+vi_v1pD_@Mcptc(y^f5yFd1`Cz?WS0G0K%T@gnT>jg z;grC&EpC9Po@P$-5+bd2T{VV&t}%pfIZ}nj-}{he~aYSLDLvg8S*P z&NLR*dwf};$MF*KZpr|mfTzoNPQgI#OBdrcOmu|l8Hxs|O}rTQY$Z7#IEBCd!4ulZ z%b>>V+W&@P1NF#3@Pl{Uh!Q15Ah*-Mx=2d&X5JUXG7%(FC0+M)$syGcnTAm_X#)+pZ<^IiHgmo+gs zeSP?IZ<$>01{@K9+8UahzP!2-VG;+)j^Xq=ZdLZM1d?HV48IYFeu!2FFxC}^egPHb z?XEYU$&_jjnZ2L>EEBaYF`d3nL4Q{HMgVl$|LJ)}Eyk*}x@-ta(GR_VcrB{P>W z(l{f;^5*oaAA0rnAAlW!pOsC;SO)B9%COtqsjgIWf7%tt5pEEvaW}Y(z=C~uK=@Z! zbwxKPIQ0}q^3|!T23_ZW@r&;ss_Zw5@yHr2s_8e7@>YLY5jU%#g1=)059Qrq$n2UXgz3 zQ<`b)uMsm16%^_q@{RBYmXVo`Ph9x89gNoyyl7<7gj~&*Ol>B_Cv0f^iaS7tll%St zZX1NuH1o+Iw75e2L4o(l>U_)$5xQCUW+z_xaG)#en8l>4?=<1X;fmH5Ucxd)$am1R zt!Sa_N8W;rk&BWQ!r=WJwP4N0HO5cE&PC=@rTiey-4YRj*fLy^C_oFuD1f{;g3^xs zbu#-4NR8I;9K$$!(RIXW??Pgzq=A@=fnIF{RXx&ui64wTl&Y|DXf=}X)arvdP(#(T z6dt>7bNbnPnyn<&AxK}rb3hK&OO3L&8iRix!wPMCvOk~^APU;C@a3}jGl_8bcJIF5 zQ>?)JCNgmz+D?s{<47H8Su2}YsVf*y`n;SRnWxhmEB4TJ=5$DUM7Xv>V%vd!I{5za z>XXQJJyv5&tGuFrBQNGE*XrRl4 z^lGMJ#=oO#Xd(p?}jQT9ym#zf^h!4SLReW)r?~#lJUq{v_{Fyw(OSvT{Lz$`s~6QSiu8 zptmurq)0vyxifELS3+sDqIIEYX(GC#LOm@OUw(poxu5{1B+~-UR%McZ;jXpv$iHg$h_h=oQ1j!a5{2x z2js#BY-Bvn3%C!w&e-VKbeE!BgKsgx;8RU);u#uukHHK7j&5NPuNE<*UOG*2(bfXV zZ>vP=C#1oxb^RTdh)ZLkstrpT?S`j=mF2lOA6yt+CKL4Q{-#=^L@J1HlgL@C_xQ3H zDFUQ+WX8F;T?M9R=ny^<R;%fyVJ-?Fv-&fpCflEQ3lGc2v*CVs8c0s6K1PjEN zM;7S?bTQaSw3UWgApl2chh0m2_Vm>D3F?GxkVshZ&S3p$k*jn@30o<1!z*^%+W`L% zF2&7TtV&cuJ+e0r(1b-e{sQ(B{oYM{ce)htcyiojurR4jJP`~F`kAJ zC518jho#nHQ)o@7!4u~e2H&FGDV6sfg^sgkb>7L4k5?D>xxwiI4Q#QV4ZnD<0pPKl zyV~n^R-fbL*$97I;T+avMjX|<4PW_k=wFZ;hbbS=bT<9=edE8)h7Nx5=_hlqhJB>Df&A^>()@`%+muC5a75MMF%-X{VFQ9c;ut#o(h2xz)K5_ zbl<_Srqrxk9!w1)tp4>B2iZb<{=&A*62(K=8srXFGr1;aZt3YJ=q{xePvk&=3rzC< z+=bCGaDwq>@#0lwPuF>krLE{_zEgQ>GkS~Hv=HVakj)keKQ&!kImPK$k=g0tS&}*9 z1Lwz1I&b3OE0($aagW2@kKHUxcuC-l<-iJc#~@aQcQJb+?mDRB_~8!^MN$gdZh2X? zHln_2bRTWgp>6EAc`POYDKzHfiVR1%V-<(@>!aD3tu>3#Tq*PQGDgq}sAK9{tLh1Y zOeun9tjtQY1qnLS+VV!dCidfRPu*EYwCoAaEx;X1b@HzCQ$9z|(6R1S$Z1xt6F*J~ zg(;tz&_-5}!okj#_a?dh<{x7?({?d@>t77q^36g>kL`g?(W9BvqnnZ&{7V1Sm%NO^ zumsM0usZ76oYkR=3<5K4_j}CPmZ3m{q_|3(f=IW5V(9MO9j1J$5%N+Y+5U_G@YtE? zwZY@11nf+^x1@Agk;D(4uI@xR4MTCCm{Js~N?o_N}4MSFwe z<1DW6gRdKt(g_!~vwjFQ5u*5EIu*Lh}f4h61Z&nf8} zYKPhbK2v!sDj#$aj9Fgkjm42~sDvCT^%9CUhfF?RO)eN&)NXa-Z(mI!uhiZasJO2v zP0+IJByvq=i9CGxrf-ssLz?9uC{po- z=G=$dTo9k$UP#y=ESjFs69QL?xd}Y6gr|*IlfOuJ?Q)%c(6vEVMa8B?h9oOb2gTsp z>DUT4IZp2~*gwu?*aLgp1l`E(S#WkUGZyCE?3B5VQ?6~lXdvdMY_my?D}V2!Ee&Fj$-(+#+r$teW}|01N>8M^F7}1TmkKJupY8c& z?|j=J^hw)&oQPj+lY;xrCjOJeGz-3pt>`nwkBO|dVnm4o5YLWsyc;VYVpa6%*$OB) z!v$53KZc5C$TdzHz;eVtlL3A^IKwRzE9vFKW5fNkE`R-LYETgykHj!b9{zl*`6D%= zdTUNk3&2$vUKF4(4+~~eC>dd2!k9Tz6675SmLabxL3tAI{nXhNU43Mhz2gpJqmQup z3$m}v{8)Tw2fIo59aM2H3T_nZ09|-K=#9gP`RUuOBY2H_4&0{Hk16FP*Cn?q3^$Kc zR#7xTNj3RxeNnm2Ux&`FNSo@3idUx>1()3iYx~PEqnnIrxIBS1%I~agF6>qn8UM?5 zSY<(`GR(Cncg%$^-;?2PJh?G(9R>b<)BD5Ozm(4Zs~3GY;{7MK9WC7O0*B^eshP8H zzSX`R@MxqN=FY7o;Y<{BR|x}ScFBx^{R+M~((jm+tF(@c@NM`QyTmI4Xl??NeUn&D zchQOIvAD)zQE$)OQ_#Fhnk0j035!8wlQSu^vgRF#;>$Z+Pct4t`VM@3?ut^Fsx1v4 zsR@D`apO?~U+jn+C;GT4)6dylPd*rFk&L4i$|%0o?t#-mG&>fYskz(|`;kTML{ws~ zP#S2=S=$9I`1o&#sJEZn-fydBN0CXrIhPi*olNujF656sxWDd-?)CO*Z=w;akZXmr z%86b60pTniFm}LZ-xC>@?H%<0LJ&Qn!9FWrB3)Ox_30ZuUEU2bB^>iaOG z{c;?7y)cb={R{7=;(HrB?+}d@EX|JEcke)tvSqJfS70_Ydz+PZF5I*+{_0(*6tAYf zEU`p8yMn55xIp?U9i#}Dm0oqoaeYo8tN*0zZVCzR|Inp|AvG1G!g*5jsmh=)`Kp`m zmDMk;I?fV6s z-L_b#S7M=)DT>`zlAU0z?dt|GBsTI)EvH`Q>IgJI{}0Udxn^xZ z`&`?rk!LXlqww>bG z%XSpquFGfa@083TEh%BCUBcx4cOw&Mll*mNkDXaAioJ&?1exTsx3atY(@t5z|YD&BTRg|DC5)^G|h> z_!$$V`sf|9ZOJ(Jm8o(F1p#SCF4DDg-Ph5!ne&c=HC<6z>5&OT1TVGQKA6E^?gWlW z*846_Hf2-XYvz{N+ zNOiQ%^KXyD!X`|9plT8pL@~xCu`&Htm<9;{_~X^zQVH};pmtb-|6(4CY3ZK8y}}T& zN%;F1p%l;nmcVDd|UE1mZ0H;I*=2Qz^=Ce$){`-L;U7xuIKojS{*ES$4KeT!n zE?jUee2mxiUm8IZ58h+F9ldYTjQSgTklTE@&=qb)F8G&jfvw0(RAP2tAubUI52ZEB zX1O@g)mHOYmAtabp*x?Fg?72iCy5cf+}&EN?f*+8<}|ef z=WFBC=~+Uz0|}Vd$!PW$cy>g=k~Yrio?eAR2?uNn#kOe0m{+&A< z>W0^y;>a-HeyjbjiOmx!CYmpLd*5@*ODxI-3&cj)pbME=dV8gR@I6W>K*PlzN~c8C_T@+3|IVu3KFe>#B1Kz7geYx;Xe#uv zDuB{()U-d4dnL6Qkx_a(>z`V}_uP~0%?}M8k9PI-#ujXi(l}P!n#)-t)#oF^L{bt| zh@Z3Pyqm`2>*OJlpL|?GOlhM6?SVn0XP_e*FXEfCw(kf4@-~3@vU!XO#d)KKoI<~e6LyHC5Fxlq;te}@ zQScq+G^a<}Brl&yAp#hd5(C3Jc5Nv?HBRN|st@ORNCG=xjh%W9lOqEq9Rge5Kks6#*U!em1^;2V!#I}z^*uBfZt*ei}KK;;X=s4PKw!bh5I~w7o{~9ncxB0LkksE9>fOA93rt*Z!Nv zK@QnQzp#F!%XBhF7(E#NN-=aN2#w^v9nwE)*6V4(!b(Z3oQonWr#t#8xPm>26drlm z-*{ETiZuiQZj@*}pz3XZGQ=O{mw7-kB1=C)iT$GpkWdFmG3>^1P)+*ZPD_$8z&twpes>wX;xANr6PXDwR65I52p zQ9Ljt>@?lW@v6EJQ~f!@RW-F^NSgYVcK2!r2&}Q?5HT^o!`=gm+w$(`>sNjfK~kTE z+?e}k3W9eY0G){H-yWfA4+qN~KA-U_L29D2=)3xX#d2lOnSZKLoz69UfJz|_jwtVsL6{0Y4 zPgQuhd2c9G1WS`M@!u0dJQIC$rBhd|KFnTFLy&TE{ihFNT|Z2Bl+xYjOF++*^DxGg zmOfMUb{9aNH6pgVxCaG^9`W=Jku`{1-+1P5j31(Vg z-dx1`n2=fOtEWmv7CL{e6tyYX!k5gMoh6sIGqZI7V#r*hJh9hn8b6ypGodVvy*z7# zo;d+H+1|Sfvk4Vrmy9~)-RkU!uXh_Q@{;xu_#5v!J+YwvKT4@xY)6Va{o|mwPrv9v z0cxOW*^}&Mn}}RVs%y35wnHGoWv9srCGPg5gkbvLvX@4oIukb7?AP{`^JVsWawMKX zr2Euw56gqPPedM%HI!|}P&ZS0_MYF+%ez#Jr?yzQ4D@)QeKbZMb%Y`PehF}_$YgNCJ+yHttkuUijsdN(f{dZ#Hr_ocfZhncv|RRh$Xk&Lg3jpJkUG%4P-(W#Si4o z1lkWJJ~7fLA86iq=OAOe09G>$=j-m@L)WpuStx`}610re7oaG*&djP1*uTlAt5nfO zg`A*A`GGPOscdENRf(oQKkbvZkfr$b8^o!D#6E|Rw-k9l^cRE8K_pk8mIJPJ%tonr znrkmVHG4G|hjK*~j@xNy8mV^g8NST7S#} zVp5Q@@&R+c(dR+})jKLT>zLnuvh&Lksy($oi{ETni3u16+x4bo%$)4e(`4h649Wj7 zbRN3M%jE5qt2Pl8yFek7K62frpf`L|o1~zS&;m<~zLA}hlUz{j3+FW2N~7{!C|N!0vWHTXqhibQvLBZ;w%Mpi z#m2>AKoV1TA>+mqS-OcSx|CVnF1c{Sh;n-Uqmr|asJclTpFXKPPt{6)pCu? z4GIXfnuetD!+rIR682SG2lt)oOmTHhlLV~3H(L$&5B{n_X2WY{U3r*nAN_Aa)u>6{ z!AV8dhNg+k`oZ0B8vT|b<=RP*Ml3cAYu*@252OiPOihZm;?$vSjMn3sXX(H#hKd#z z(5d_rg{fK|0~tJrjX6S#*oV3rqVTa(__-cB%I?WRcZlPL%YzsiMv5!&!2nka*99^2IHn~(H0B+T${utk4% z1+hHks|&IkY9uADlbQVU)mlGVFrDGkm{761LVO{h+EGJZp%0mq9&g{-)}U+5yf5!y zxRzcGi4Obwtlf)k)V(OlYIrR;9aWc49_n&DzQGlA01CKU!u3vO+FA-oTgv}?uA&WS zSeR?JYn=sB3WAv?>KAxQ(ifjK|Kfc-eYxdUoG0*+d(zy6%uEdQ`Ii0&K;3=2QzdOZ`v*;xYV!ekvv?Gafj zUw0tyGF4e~gY}ae0XTlxB*`%e`f*5>CAi=<5 zlzMn?xdQ~6p*L-w)fnJA&w&a&lR{F;Q!-XI*t|shJ^~fjoLn#MX^>)pcE%RPK*xQp zkr(FSbZj$4tv%klu*-YKt01Spr3(8IG9H%3IG3;EWP>51#P~zmdY27??}`L4{Kq?> zwm2XWhfNTNxp4%uKHKctSq3RmL0~s=b!}Tnwdm9*D2n4qK$YeeUf;wswD?pur zS@fPpdl{w8hxAsl-Pxhc_@9nN^G(>^YC-E^OX-Xsq|r8}K8v};c1r2yxmDYuO@#w( zuC~a0L;9T&1=w+z`I|tU%>{O7V;NWLiTqA)0@sXU)UBqH5H?;5Ly(?F*4^w4+nCzcQEP` z*KDLlly!>5s}yfoYvN-gYo82%hjLUhXV*|*R1Dfgsa&UTB`=Aku*c>+ zvHA3UMbiA$(8N-6AGZ6TZ?$J8_g^>25Us=p6pQ=dF0)=`;c9k|L*lG2KR62JV)TsD z+^`*kyB|{p0BOv1)D~;MGw?}dqw99Ckn7dxFSh2783&@%nsA|C3He(NCMJ}UT;R*p z6DB3vV47}!MJK4OJ#Gz$=B)}1r!!rSoTs4X9ZX3#dfhkM=@jGlAcmdTd~cjmZ(tr7 zN)-fsC7bZ}$VI+DbrmKscBN)K`?|FN3V*i$@zgY;wQw2rNX?ZBx!QFw>Q;@sr`%T5 z-O3M*Rg2Rb=w7tuxh7X-!g-Y?JEi!6z2j1;`opUbYHdd5f>RHoT_`j2*t9dhw%|7% z!4{M}t)7cRlt1xOC`9?J{aEMFf5=cM zL!^O`I@t03w5S14&+mh2Sih@R1-Gx9MH~iW zdsjY%ktDGufj`FYOXCU1ED^`v$5&g)E$~^+v@F({s}M5NwqLqS;*)d!NFrDC#mT&8 ztDULY{*R6k)Q5D3Bpfg87y9r*lZL4RO|?jsa+41# z51#5FYHm>GOK7Xqc%bLYMt0M`MjlHCZ7OKBP=VIoE`i)9m^7EIWM*=l2%pB>UsiVN zA5x~cW_RBn zQeIP-_xAh6J6!BoZH-!CK^+ChtA(L_%w%;7pLZTu*2q&0o&N%{vd7#{YnTodMupAh zx23=$BC(hXqpd)GKdc(7N90Rkg+jfTI~SnE2Q&ktRmih||HmboTTE$PeVDG=3HR6^9Q2Fn5c8eiI z%;Epi%}BBGz4+&G5_A>pX+duLYX&zEkd&NKP<1Q=P&e6nBu+n7+-E4y1 zY;ec>h<4O^%p6@hXlzChy`1YjOtl%eup=10{Yhbs7rECAqKmk)W0wIY_MJHc`uFii{ zF{=--V`!-}*WVrnzY}W_E45j;9@@o=^qWhiwiMQ?C3Ks;L_kN2H}y(KID9}>U{%o7 z#Rty^3JcbydWr@9rBzo8a{gqHOi~{N8it};pjAy~E*RdJgzEj_=CNK)kg%sjZ88pk zbYXNA`_Jn=f{EHGF`w(A3>nDg44tk5@80`L@|}*jv<{78Q+AH_Ojkr%MTO6rcv(1) zpQUxRFDwmMCdo5KL+IQwJ=b&&X}wA7h#*1SiPwKhNu^ex_(KWMZw${AEi^ol*zo5-Xn1cdCGl0utL3fVq5 zE}2INL^@NQ-$tcO(H3l=S>Z7=n>Tmub!r3*E@AO0!oV zuMm}Pz_dU<4dm$GH1Qn?2nXZPS5XZlC`^7!z>#{?p&X!8cxCv(^Cga9Ci$nvM0`Bp z+=Xg+Rz#C2{_2zPA9Rq#dIkqB%0khvKlvlCVfSB0w#YTK-KG9{dgY%+&^wKS5w}5r zz0)%~M3dF`hxdC4idnJcw5zYQ-O14%l%sfD7GofmU4!$mkeH{RSe84it7x7aJJ^=w zp0^Wcs&|dYe1X{slZ9@r=dlY_nmyxEg^FcqR&F~@r)h}qR}ns^y+Qn{9UB$Qcv0im zJs?Ik{K2zxph6nR-Tm!gNjq1iMo22icN)%djXC@xkk(A(;XYRmU?$JUBjy# zi4J{;mVF`k*mQFtkzZ1yj~&|9>Wrgk3IU_DB2l8A>I5&(I*+e~aW%36hnm1zem;cw zR|UWHRHfvFb{SI*Bz$hd&=C#(60|UkPbe7LinUAqrG<}wKykXE2vZ{bD8HpPM|}*J3NF38K6D4U&87l zJWm1zZ+^0s9#=vA6J9f=`gzU^6{4+6Ny)~5q*HmYd#;m?(rv`T{;#@fMSR_Kkqruj zZx;7yOPAT={g%`fM`(7I>7^X>{I(C z%>&6h5W>pQQm#W0hb;}bm1eL$SA@cyN5eN<)E@E~2jBfsgXO=)VqQMlYfwx1exi~w zcL9<}y7X1avC|_%l82aZqpq9C7w2b`$|Q-%cJcnq-+BFUQBD2?yFj+)hPS9Xx1f6NE~f3x=WeB7;!;QS_aYc%lz81HsQ>(X$Q}T9!sL|w3FZ}ILTr2)T5OOBro~D z0Jt&S9*QP`mk%AEphJ53!SBSBLvG16d*@ZOJddO*9NcNapfZPx1WRTK<@o2>WyVel zJ;ai3)qvf1$(m4#b9gn&TWzR##Jr!>b@TSL{H*`p5eWH#YOSjF?I+04#=y1wQ31Zm zop-4$-R0N*WTy3_%>8s?$!cc;u>51rACnS**&3=NE=A1CsOOUlfJ1sRF^m`5>zaHV zoE;&)Ijhq>y1(=Plqlg)rWnMCihhZ@@~IpJh?!vl&q(VvmpUKo9>q}LK>8NRr7Boy zgYar9lptn5-_g#q{}cO%pKfxk3AdgA(~v+8$o#ulXSmzd-xQA8C0i249uF4)-ReNh zgV@oIoUri?)enGDf}cJRJ7SMe>9@dD?_NpX^kf^XCU{T_b-b$dXoKkXqC@gr1cCJF zJPzp}pK8}Drenu6!i`1K|8!=c#KXCKzJ%0?`u}<{8bkb09vpPPxYrgl$qj0~F(Y8Y z$j1}U9TsWMPx;QuYCGms4Olclqxwlvs$O^>Gzg>ls+Z73D5>;t*GVYbvY0HeeCo=$Bo zCYxnU)!}aZ7-|qMLYrgY?3ciVHyEZWXE}rQeJD6<4;Eedz12yiS#0o>OK)t7_Sn@N zqi>KN)@L8}_m|l!ES1GUF&#Ey^zbLjxX!-ku{3qS$zU|^_8S78^>%j)BFRw>v5<9I z9m9A@9qQqebfx8&0{K5=t@ZNU9Oy~x47q7_!o%(`dly<2+~qPE9(+N%YFhE8O7Pyg zpQyvw*|;B3KjmFaid4>hEo6P9nsRrKf6bg{uf7Jl`}-+@t4K zS1S8sV`5x@A?O$Z@OIIc8C#n6OkXevJwy(o3>sKVuzuG46u$+EJ2|SY&|z)l1X<}X zbL4ieJ*T}f+7CS=#YqY+7Xbz{2r+c+D0}w*3g%;HycuO7xZw7vDM8aEB_v~s)Lf2} ze95Kj{=X0BD9moN)_2%EENzOQ;^*h>HmwqAt0qnG{)%&oUwCzF$P)bCO1mEz)t|l% zd3aWdi%V!ceS%qPXchP#IRTWN$dHgO-RNCXV4@MtO|L(7^EZ7+sSasfs-eyPi6VW3 zGZoSMl~O;Ir$jbHFjm|OHSnLOQkv&qe{UtNbhCyG(0Aa2Ta2%{IMYZ37n7_rvTT*m zS3Y=xY^{Iv6yyRHxDe!qbjiOf(>a1r5~Cw^OH{rK~_Rl}S<& zCO}TQ;cHI4QA5~|zkArOLkA`xtsQ|BT4dv`vGT#9X{ljk>fofS^BTf zet>&+0)RLZ6zPvkz)Ee@Q?eknMIlrCd4&k6j5J5IJeRFjpl)xh&>hIW+J7oy;W?|K5iZ3$y4-O31sKc?xY-cV(e#8 zPC0L_j^n>Y=k+p~nPy=MbFm74LsY}w1yIa!HAvu>K*H4GPDAS;0gpLK-uU6mlbyQ_9X*0Qb=C;A1~bF9W~NFx~mG?t^Z@gM-v(0WxsZ<{J_1R zu+(r|!Ey??Dqq2r6U1v4hrLV-+P?LP?3l89wr7~aN!LqPawUy)+n>aA{kv=}od1zc z6TWnvkj@x;&EHfeAJpBnIzRu@;VhnGyM<$77_^*#x)s=tP9n4@U~0FVxA#9OCK9&2 zK^{QednILg1$WpZcXus|MmOKTVcAMApiwZ!I(EDALF%e+2Kg`fBofso3nb8JZepqL zyB?yqnv;g&tpVTby)EU)jli+Z=@y}(ALHbf^NK|9O6s9c{eHYpof#~W?o=Xy-=)`G2Y+9|*BF0@rrIMbC41cr-C;o>!E zsD4plMF6$isd!-?`Yl0qQyC;$w|un7Xw1+=6H^@rBTP3XNV!98^zl9;jz^0^YsQ@vp?p8-3)i>WyaWE)rLT18>kt=H$|m<>&HJ)!>pF^21QN47X_` z39m7h-2ZhYsLPcS;3-2NIV-SMwzQ;d;$>+Y(lAihIHE5Wem5gC(_fNbFY+bv$6w2b zuU>@&o6A4W-!jA;Xm7d!jac-eX+>1fC4)E3 zVG7h>{q=Qk3h<_gjz8+LJ?7$>IAD~*^p)Rc^8Q1K>IV{ebz zV46hPrwa})tne9s&_<%^l%f7Ktdx_$XS+)D5bY%5SxfB^4)WAdsw#95UiC}tIf z6z36aijSnPTqw?eV=x{dikW5pz^#^>rdqm?sSZ>j!9X8-rln&? zNlv7{VoGgY#jw1o-$~p!-BeKxN#7T`=6orhnE$kb@SarP9qJtNk_+Hbo{VT?%C&V_g?=(Hbd|Lh7ARGw$i;`FbvZG7qgOdYSJ7l-cjbLwZy-w4B{@-&g- z^;ieP8PIOoViGl1CX`j~80%wXCsyN}!i8(ZzPzoO=asSD7en(P0@*(mrFWbuM)EEn zW!N#jG)I6-agzNc!%@jW^ik;*09Pf(z1nzeJT)^SOyPb5tVN^ob{J7b2M&^{H-p7x zjfSJnW;xJLwYh-QS3HQ~=PYe%nO~t|*YTEwPJ) zStpMzD)qFgN=>p6uu>k3HlQIbL)PsO`Ma56(g5+ZHYV$Uz@qm~mbX$h1*Q zeMMtcbqS^*yX#7B4G>pf;mSmx;#rmScuQh zmKMIRchPv-3}aKD#7~-9t^@lz7f%PR4S_;EEHRMmDVjsp8nD)mMd(7!Mnd6Opv@ZJ zP>+Ll#z{Sdt^QlqKufiTVn1m|Ibx~=*InBKVVEt#6nD}M-Gu`V=Wl{{G0 z70lUmNeT&JQUo`o)_#F6HTol|EROV}l3u!{;yV?H`wdJAevWYa2ooul76K&UD(imT zAJboo0!5QIt0>G(Jz0E3 z89qR>)18&8iTbIcCnx!LpXS(WZxN9W?^bM;H6S4@xyWbTWrdxzDPxJ&FDi`a{6ZmslnFWJ7}OBZg#M4 z9yLVZ*ZC&!VU2C9(gHKri34@7_OTG>Ki7_YWBg%D)LiyeO_^aWie2b}^?`Q=CL3D-=Pu%)*=PR=5%_aQsMYCXKVJC7J$|KY$}I@ z)4Q<)^RR(Xu0PKw@518VCdi0r2&ye^`D8*QVC-IFoQcl})ehMbPk(x=Cmy$JnsL*5 z=qCqRA4V!j8ZO&<;3^5>CrJi~Xz>mWMyG|FuT_l$#GMDz{K;hpU44GQX`C+#6l2k# zN}#+fG>oTCiWbSvmhz`zc#QkPa9;GCVW~5*O0cE11zb#&ppBlvhsV+I+NmwHY~5WV zhJiI;!`}(gcAs(Sd|rcv-q2s0*>ZAelun9XA@ED%by5LZ6a(;rsNstwocP-1wl;NZ zG`bpS*m}IxbiJgFHIb(-2Q3hQd*U1v38k5(u$N6znxN4aA-MaBt9m;Zr~jQ1wsUBCE0_I}cNql4vzK zxeddifAG!pz#No9549(0S=dHIZx>WZO}WjND?F@g1iakE+u@@gijqvVv~tskSI-bC z2eo1H08fWh1Ryr=?^yEHuCE^|W*>YjbzMEEynL5&d9p`>yiG#~tAB+!xVHs_`@dOD z3$EW)pW(fxhwd;IWbtCLHBTsqiQwnhL?YjAm!Ra)3DIAeXQ)$`JL)TrE0F9cCz-sC@gLgyauQzU^a%T&oe&EGxfCy{h|%eP6_4KPp}s$R5Qj`ZI;a>oQG_$aTH9XbA3vEcqqhRhU%y6gQ1wJXE#S+FG6p? zvsrDp@NMrSfaQN-nFQN0Qvq`cuIQiI&DSX$)&7VTIy&pfqz;HLJrJaC!;fOF#>iK~ zc@bEzFn6g5Uu;+=a&IQ&6Ny9f3J9fjXCs5{i)G0E>G`8#9CbwU{AF(dulVFd{|0!m zRy}Kg>8iHMCA#-e5V|$UrIwRa^iaTdI)GI`ZV44%`bOA%;buy9>B^te9rVwVd#`2y zrtKD2_D5cN$`WfNV&FaTFUoF4*Z7?O%`?V2+UzygIiV$Ds;vO*W_UpR>u)zhcnb?PatX75ne&0r=#08{!k5jZH=F79oQS>g(@7Wnz{8 zF}X+<82~Y#r)h73C;{OH+9s=j5aj z+pJ!fL3L4$Y7x>`U&Vgb0W96Aqx?y7Pil@X^^WvEN->4C>a0x+bj%5$5|M9Sj*QZcXq$^t`kCHi3~gCa}MAjA~b=Q;z@s%l`nUK8{$`l zFe=1gs3e`G6b(#86S#KWGC=ejlIYYAw+HG9E{6Zr=}v+%u)a?K3U;lE)FCU?bg#eW zx(5=k|2G^t^l4{HN!FuT$!)tCQG({c+))t|Ag?e`6AtJTim3q}C8+vK_%d(gj7QaB zMR_hQhp=k!DjfYj^v}wbIcomnnX+!KXWb)c=cSf-aZc}S#~CQZuPmzEJVvEQ6D`5f}h- zS?>$l`AK5`=n&$NGJt>P&_Q2U?mf)cV+fdxaA7*Xpjh9q{LWuyQ^_iT&;aG9%*k)b zA4msFg<%a)#LB*y;+wL{)K+Q>e~+dj`bxBO+WI9?0A~+xl!jWztO3M>x%DDMJoPAc zz-`JEYe(ga(6#_MznTXPV$^V*k%kK@65!MPNLwr~k1 zJ*Z@y%$71^tKPk6h_b_v9`&3`nI1&5dXY&rL`GTm`DWD4CgNZ87$1yY>~VDvHlnt( zLXF}l$hHbcMYB*p?f0|4{d!NH4mW&A2E8@iRoea&DpaUf*xANX!=6Jm|4C?%l3t!y zb~R#~L&s9WAyJdGA< zXuvBg&u9Igr75hj^cis!@hY(8ZF(yq?<8&3;3&*~SA|Hk!zv(HJf9otN-P^-Bs**& zEpKG{8?c654TpN`VC1}^linDV0v9=LHefGo@*p8cE^eCWJ73)IL4}#$dyO+}@Bo*d zN9WFsQGs@c%u3#1ThT?r(%HCPde?{4wJeAZ^eFv* z25f{T_ib*lpiOW}BJ}c|NG_A%fIBB@H3&bg2PG>a^tG7 z^ViGM05#bM75OzadG;7B=p#2S1mqFrsg?{~g;I?jKrehQnk`hg-MYnQ{hVd;-`n0f zL3>!#obBE?HFb0UN_uTta%Pgo@Q(R?=G~L&j@iHrxxDPPKB|``+;tsBdeR;#p0UWW zcL_S+ex7%}W$r8}Vun0SH9e7C&)W3emqR+Ka8{Qf!CX;g%Sax{DS6^x{tdBg_}ZD% z1S>W{t&Aq)hfkFCgtCbn0;;p$k3QKiPU?Usw}QNSj6lgpJep(a$fs zl+`G{+HYBa-%5spbfYi!kud$tfQ^;5QlqGew&CjCW+}fx%2+puStuI0EeisV+*4%^oCHa$fDRdXw9oNu(TO#dG*wn{j-Nfm zT4-1^@2UG3c9m8FHJGbx5sa{BfW=&=NfzdI8x-k8;Sp+ZoU`J{Ai%dmqx1C%B~y~~ zi3sr{K-COzM)cl^*%w^LUpgsg=PfOIm3!XGm2St*A`+HfiURf-5IwJ8sh^~%?&20K z-%9EWpI%PByYOUV!on3xgXB@k|lx z+)2DQd}P!e$n$E+6BT?YRcd&OWlnHy#{|gjD_8nIwTNj;Mhq}T>yoa2rq<|7P(zkWqC%&$Y!C=2rPw$PQLc zpv%d&XWh{|jCXhKDs`Y_p&e=E(w$ChPlOr?}~EN$oNPYb=h4zR8H z|NW?PU(ON5du447Qfje0g4rJdaG0QIUL7*zQpQMYn8ucsk-SMco3oC70zMZfIE)|y z$0j63jh9Z+rN>5Wg~?IqtcTmob7a9pL~xt-fso=WKoTir;d)TRjE%I8 z4(pF+uWgVQZt=Z;6tA zAlx%#B#>RH;JuGF1c(#Q+fcfNsY|ge`IKmbs3jsr!CYiuRHa89xc2*guxH(-8?EN= z6N;P^E=6>e(}}`}ybr)=h4W{i1Y`VH9+D)}pA6~6-+K6s{GM^gU{~`2U{Q9m;WQF1 z8V|;TF5#+2InkpLygpDEL}qC4fFY=>VCg_qF*pFtX2d&w{og6d)Cbz>$6CoPJ@drL z>mkOI=lebPOzNynuG*hK)x;z`xWlr{V4#;oh-(28xrN}+T$<@m!H$&#mmr`xUQU1& zeq!}R&nzc9Ic}Ss_$zCE82Pc{cG2cS*QZ5GMuAN44 z!i4Szj&AcLNGv7wZ{Gz;P6Yy|Wb(d5krkWJv;kxUPORbNWH)wcg!w~1V@R6MY9(YB z!;?i-(-ZAvze1nF!Qrn?;1je2Y9014396x*NuXwKJj9!Ov1BtEK_4KRxHH9v_q4z+ z_!9Rzi=cNHI+Hf)vs4)Y1=`>XC+4^05br~RyLok5ks z!~S;Nji{>760dQ~ac(sijR85Kje}VeZno=0WLcv%(o-HN?A@$CUA%;UC*&*<9q%l{ zMatH=6>nM&%tL($=@K2~pDM&)TR9u%{w44QxjV93@K#*uZ7v&8ijXGd=?l1_>k*}H zlSp+~`f%7&4CRtA;iXqu!l|i02kcz8CKfJ?v?&MFP-jFPv6&G1 z=8kov+Vm!;hy)RA8KfS2Q+(LCJ`mOdOrnU;0tWfZ z|J|+gVHCe3zu(C)HU0HHeV?&B0cdzewW}>Al(Zexo98-V+xrA0@r=6Y7Wcs}(HR6F zDzBVW@|4Kx=|2e*>qv&n(u`aUHQt|I6a#%u+KhQ>94SpE#SD5RqZOyA-XRoUzJq}L z#*P??GGOD}t0>939N*K~;3)P;0iN!KTV2)5miCc+mCSE4f>st+RXI?Jn=Ex+Py8$o zJqY4;9KBY4o`vF6W5Iy8JY;ZGP6$B&DNf$fJ6-uJ!NJ~(p=RRXSC))SqZn9r?+z($ z`M$(g_|?P}E!=x&A)aWS&pDJFGI2n{GOA!|aFuTArw9U4WaNQe1_0 zed)LB8IT-zved!tuRd*J4&duprJFh|8#oaE^}{5RPY`piRAfSshuU;M?Nq9Fa~jygnIYzTu7!B%hQwioLU3FZuOE*s>N38bJ}x2N zR~6hShz9KaB5U;U1QO?0bT2_knA=q}0XpP6FYoNFE`ZiESQ8(r5ls}CAEvA-q`{C~ z8UMzOQLE}N*}H;tw$HJ5+bHR6ksk4I785Sn^1p$#QyJZ>*OMUkWBR|#4!(?%gLF>| zQoPcBYI18I1bS1^Df`5b?V5Zi`soh1lA33c_!MJ)lEGbg4>Bi1m{PQ&LJnC6Xl@mK~FDQ&5{>U5fa!hdg2jyeje5y^$l#tKpT! zIY+ax_Plkk)u3}Nu)mJshqw7|yv-@o)t$jvGr6b#la^~V;kgS*B=#FTIwQ3B)Z}Ff{EQ)~u0Op)NLUi{H%fFi`fQy6Z zi0_c;8xkB_6F=lW)6ANV=gA8I;KGj8a!vtFB=x!2C2hiE2_!E|&d#P>pUZb_|q+k&qYH~G-wxSqQf z3g60R$eW3)W(mibYmtQ;CnQoO281IvoJyR+4HS%yY6~uK_8n`uD|ElbexWlYn$-<5 z*)r;E{OB$MiB|o?Wn$4{?u;w)=(Ry{4Xz-441&n*3>3(j;G-s|C=t0`p6AZ z_m{Oh^Zpl&1-w|8VJ~D$fRFc*B0g*g%mifrdrlB=LwO4zAPf?`@A1}mQ>phHyu8s) zI?TqZ=vm4uE=nfGHWzreTPJ1KU?{WG(Ic_YkhfJqgH5u4LURoEL<^E=c0}+eps=#M zflU=p15=K`eGDCQuLXnV64rk6ix4p$%oLprM<=?X zk(S+ELO1?=Prs5gT!0y#q4mz?c73zL*1r_JQZz$%luI9#?<@E*X$beD!4Kx+P(FeUR+EQnpx7vgja zZh>OnAL+yraxy(*;pyN$z3@&+{x=JZ)z~hINAB3Zu=YMeOK|Y8=sfk%U+l;e1PgHE zI;x>H5E8iI4wFzSV*hMcd+|e0YfYbc26H(A znkW!gUqCQ@aC0Ki$cd#4`tWjCT5cb370e*FsV=L*dXm1qZ_Cw31fkb&TH&Wk1F@w3 z+YjF2P}|N&00u7+0X8fa8~{(ZpsUSx$Bgx{$hyn?b)&{9WN2hjQB`@jbN`Ze)~j`p zmNKy&71=`9U~DXe^U}jn!Whdje6jh$k07;2r>EGoGdi<@Y+~h#8kJ<>P#Ak*w#Yu# zFT_m}EKiKD7-$DscONo@w&-|_S^3|@n}|zWl6b@T2vH27-R&YYT7M<@I+DmOjg&xj zgnIm(rQ4~)tU-4}|Ai=JazP=60!GV0vX?8IG+|ZlGc9vRZjLcWNhl{J`uH#P5-MXa z9BxoH4}VoNvAOImDwV#e5O|Wso)U;My&NvX-vjbWz;*CRJ0pce)WNGTPD=kw_Ix!U zBs+R;C@Qu1A_i(RCfIOV16atWQhLRA?+VOkb+(528bhi78N{`(9tzDJ$q8~D9CeU=j zgr4e$rbp!r75dh>3%T7efIsQTvT6nu&g7Wmm7148AO9p5np#_UvzONOrhIX`0h31T z#*vc=1M%Hkv@d7zswc^Ql*0)#*R<9L*J}%z*QZ3%ZxjXJ(&194?wQjuLjC$u4Z>aB znW8x~!gsEfi{SavY_rea<|g|NsM|E4c3hb-kkQGC!gDuyw8KexQu1#zafF)L@lQu? zaoyIOjDY`quc=iMF5Xl*UI=x3XvP9xBx4D0bRU@b*0TNHr`k)s|B$4EP zklkQ6%bh~!FeoVcIdFsuEIPe(__?zBbtfQm~FwN z@oI0v;~_A9HNzbHM0aw8GH|eMuL@L1`U)~*$WdYGX^l5u|CFPLV{)EmaTmL-90v{$ zX_?#B3%U0Gp?fdBx~OM%9)-IT;g=_RU12+#F?}bO3vIYo2HJ7je;K@$a|vzBrqf~> z2j5dW%{399j`ZwM9kW~PJ1L`Rd-^MMP4HSm%zr-UE1d!ckvHt$wfQqcN|pr;J}z9} zkL%)TldzE4ey<`4tqVVW4re8kHHJH?vhFZ~mF)4B_9i-x>*H>&oNLH(GFtb1wXANz zAyCy9Kxk@(TfSV5(=JiPAp|TP*$>XRZve*jt;8Dk^^9t(xooEa<0rn{Ij4z1QL##5 zS%C-M(-JEUQ@kzEY&?4%U!Opz{Vhv{awcI-_tp6&?A@TsmVzJvWPme*f}Lt8EeRG- z8BZQl4jEgAj`$%aU0^5Pul$$#xW^tg)r4+xGlNz)vc%)MO^)cXS%Kg4#Dyt1LAA` z%?A;Q{#v++nx7ihIt)h!O^Bom0AbK3tAMW-3OGuGYAqjqRZv`~S;7F|chBVUch-j% zrQrLn82VMJh{i}2B&^h+1YY}eUY_%rwn>}Y7sAL8ttfcD$&btToq)fGFxSj=6Jf5x zo)!m>ei=;M$A9!keaM-0Lq?(3K>fwk+*mu?)-2|L>k2`aASxB7B8A$}D)Qw$ zwtQNOZ4gtKL{*PuPS3K@sJaU(62x+J;N8Oocck?aEi^0Z{?)w4sEyl6s*u)T0>-g6 z_}Su|=AgrsAWp0ij~@hOW5>tmJp`DD^^WNJS)ln;UD}tP7U`QcD$m60w>G)Fo2CXjxkxNPVmg|LBo3m@ z@C{xy+1M(@T>&qz1!@L)6{H#~r)>Ed{ZiJJ%MbM%piyHK`U2JH%$gmT-Kty&JUV@seO zm=8z!X6!{f8Pq2d-1Dy)L>47+U^lS&eUZ6TdB`=eQZD8Ftw8niStZ_;?AZJlefBLZ zwY^?OFt2voZ_BemN~!JV?05fHY?FeR28y-_7j$i|#)H;}@uy9jkHwTRtO;a~7|X=T zB+<4?Am zu}>S82}cte{mdiN@fu`{t;X*wCb4O0T>|s|?3`ID2+BxMNrrPxNdg7u%x=h#MeigV z6B{Kuz*7(B3n-AK^VD1?oloi@5W{|)CA^kO!59ncBsG9cY42g_2{3X5lOqkIHD?&> zOH6Z0t!Ri=m%U^>*lYXTy^df#BM9^QqKfdUaqqu&ayF0^NZdYsn@5*!Q9v?rSed;- zZu>pt#%O7Ig?~fYl6P_0cqFI(_CA4D7XmF*TsH8&o&|o{0%8^Rs5-MP$HOV9rwkyZ ze0hAYZMZmy$J~RnYTOIE5mSrx^+eP(K^39_wNBz*MRRRV#a5?^t)6wyp&-EaVPq}Q znGM6BlZfH%gQFq#0dA$C4Ui4F>{`F~&i!x%y#smEao7YzpYb_x4=&XPW$>YURI?@F(V9%7?76_Y}F0z`^w&f_*O=L!a3_#j_e3A&HoY6fT zFL_X82Rx*sqM)db2Pg)ypHQAffK^CLuv-fzYQVh^acjdmBQ;9W3mtZ+v&{419B@Nj?;A`{{S zC4-)FdEc;D0?VwoY^EC=w?xzP5QU6R*oBMm#yQ^%<=-{zDk6eLP>l6AW-5^-AVyUI zj>Yfb-tfhKEDBDJd%Vxt%cVQn1@2gjrqo~e8N1)j4|xf{$M%6g+3_0;EP`RBYYt!o zA~?HM_rrUmuF7y+S9g<8oWZ3kEG{xV)O@4%4zJS_@nnwb=2NCF0Dc`1bduCpYkLsU zE6_a7FeJ}%dh+sGDhMC}R)l5}eM_ZPZF`JYDIft!vv87+A#eYFD(fh~OV){xi;KPB zqY>udH8%fpDy+3slhKo)zu#emqR-QI>;ho9UN`oSmX2U1)rW_={*Sa5stf`oo7TIZ zeEW!j##o-&BMOc?3HZFq!z$NJuS&dfS!1j)`m$2snPaG4xZ+BT)Zh)16?EkH*utAm zB%x$-N2frpHww*x;(ELa4P3fi!AM)CjrIKny(idwr?(rMJ4v{n??pCYKSF>nb#RB3 zk*JdM3cxNt4)lg5As!I@jLQ(jAhIlDAN+AhT9ooqcI#eC25Wwjz{y$RA;?qn^slJN zV`fCr5P8M>U6y4#zQ+L*OcZnuSPu=OceKcaM2A9xu?lf!hxUX4Wby*YpT7-Hg#G`w z?IQopE1GEJ6P^Yj;XKKbIoJCPYqo+zdIH~ubXbA^_Cylb3rR2d>C+3@fc4=l2dUeK z^Ydj-vcVH3=xrWrEo+dS@%GWkATKwIH;C_dIV{(?o?r@T80ngQ`nkWDn^pPi#uAKE z=$Q=Ni*ph|*@?b;BV>7>jg#kKs7v>_PV68v*phl|501gGW**}D$6;8s$NbGETYUuq z`OG!&urQTT<}ELET+;(h`EFceQzT6aJ1MiMxz?5?04r~WT8uPkm8NL0x6_3f1x<0M zvEcYnoGeN(SO;#2S&?a%fm#{IO9RXvb#kAS|NphemQ#JZ5%7t(n!3s$tbmD*U04_P zvI_2@8S)VlA^;+s3BR#DmJ2H@RFV+GttPqDi8IOKm^$s=M5}lME$$egN|)IGT4aak z9di5s6eThOWIIp z)a3VkTF5%U=Kifgvi1fz$M0QsCFYP-Cn2Uk1`N83rwOLzMy{4e60( zfAzF$oHs6`a6o$NaG}8{J?-cUKk7p7`dgw!!ijy}AZh-(iTk|;#W?V zdnhF<_hK1~^U;7dT2-jIK`3C~BC!2PqX$r6fY zj{))P!?CA=qq~3NLBOBx&SRe(!Z4NGr-?Jup635?@329&ym}pcPMhX zDnw2|wzX(@%@7mMg$@0y3wB-fJ0+=Fx6_zi-6KLmCbqqSYG+1W7E(#%+4}vERs}Xa zIWx{f=R@RimFG&bX4=HU^#+@dQVc6$M*uHC(7!?JJxuOrp@fZ?@`BCy)&8Nq2O2?m zz?Op0NU~y&D843LDc%m!s-})OUDgOg6~DC48Cn@u>fdw%Gr52KBY?I!jXW$ldwReS z8Oa_--ZXyCe`zt<`k~4z1gBsS*VUKDSnumTI70Pe=wZQI0y1i(Pjo8-z*ZGMaykjq zf;3&vP{0wzcv4;;YdfL+sB`NV1c(=orZK*xVujnDo~1jI|G%ctzl&TYUkT%2r#krk z1Y^9DV@QJbN*=bNUE89xar$6+A7kbrJ7?wtj!t(H7hY7>`%2Nz5A7jgnWZgzI#;5} zzfPg`et)w;M^H?V+(25hzSQ$lQJq60sflrqeWUOwXX#Bse5bs$sM9JkkDvr{>MOxn z;&VK}DOQey@+(i2e_hJ|G#|;7XPY{?35~>2$%7JSFiA_Ul}jO&6*~s0!xEg@j;a|3 z_uKE!UNH}9JHwZRin36LKKAzxyVy-Y%CggxW zBn=tinHa1y@Lx*cXPHNVFe2I=G4FZDfFr`XObDapw;t;0>R@?=R`{DF5C)Y{Yh8?m zVp73#qb6)3pM}KNZCh7LY?@l}4r{ePvd8KrZfCmQM)K8F4^}q_pn<&|s|YC$owLod78Q?!sKJG}HG<8p!T=bjagZo5QYbTH!IRG=%@@sjZ%bd2NqSP9aLL;n# zf=iOmUfP)kcg&h6N!JiV2LPIEALwak~1_jT8ELSVEE_42i^Jh67`Ak3`tF^Qt&GJx)y~+vCqG9I&7OT|=-@ zjjNT~w%1*?*Jg`&^<;CuikD07W$!?}6(L}6lf)A@qo=#MzPD$N_dPWKUnwoanBrlW zWks$&&Q|;Helz?xBq&6mI^3SxG>lEEIY)Pf%zNy781-{aj^lfZ6~k#n&L%>Ka57{E z_i1icR=Nmw7>;X`#X=?^gkz>$R%+s#GBu+pJbqyh=--Ufls^(#TJWZ8?kgofY~ zTw6L24#SE&5yRKQ(h8!$<(@WsVfuP0!27ACp$=x5rvrlF9~uVTT};-0%i4=ljY2@c zi9TYq8(=;OYYOX&^D@!Ne~>g+uaMMPw5e+ncOP?|m_hnjVD zaSPpjbQ^ju&aTW3^9f)oa`P^o)D4w}<4e9KL{vqW`aEfdfGr^FI1NA(2Ok5Dd*#*)RrXpGN%99pe3k@9PeYS$LjD zK*Z(vs%$^aDS7XW`zLAas5ocG};e`QfMWONkx#UdZA5@}s90?L`jF z^KI>b{zegl@{VOgU4T<=tNZ;t##t(0t)cRdX)+ko#!k6Q?LisuZhks=B9y8Dc5q}| zM8U0pQ~DYt)byMPw6R$HqaGTbxQ1%zZp+o(8Ys>gL?* zNuilTxjcfo;)i?S_PYN1aFVbyt~=VY>F3qZqO530rC1?i^@(XjYw{AIp^TCh_Q%Yg z8=pfLtBABadva14QN`xHf`t5a$Z(T@OeMA|iK`!2amU&4crycL+D)Xik6Hws%6FX0Ls5{)hqEPeTPm4o02)Ok5%ug&mU&Qs>lO8rS z4I|)TCkHX6y15TIAHinh-LoV)Q36{hXaItS^ci+Yk4o#Vm8WM=Mg88(ox*6>E21aF z&+3*dGe-WH9hEfM{}pT!At3>>kpQg_Bgp!s(8{OHU}x6m`0bjW(r9Em#$}Mp?;6Z9 zC~KS}_T)`yz|f$C_r42%;22W;KnBbIXFf=Yio`ZJtY#IzBpG~N4uFfvmPZjn)CIB@ zqoysZw1p1oZD-PwKVmV&;SaXv(JFTTWitF!xERiS{od_{y~xu&0nV zmKg(FkLeSaBJ4ib!XTF#miiS-I3h@Fq9}~rZ}loguvHG$r+-5=!KtC2Wn$_|UOa9o zhFha9w>szI_Oj84bSmnp`s^M{QdhFKVN8>1?+R&`UJ_Qz-{9ZM?agECX2O95WsPBg zq&u%Ke_IJi;wjuo#g_J9)qw#cr`v%BqcvQ6K6rKuYK7Or!88!IO|ap-4;3Xteyh@g zH+B|7nG8J)c?!jr$6G1u3oI(!*sZ@;+e&sxSO&UHV!=dH`DA11N0emUGO`te=6I3k zPYJ_ZPRJa=kI$Ut6ldPKBShI!#D(ZOY`}205$x2hxgi<$kcRZ%1u+93Jx*qqD8pA@ zAZkd-u7SJ1>mHL49V}kRn1Hi}FfMB-)~2&n_=a3;3WLu5kO=oElbGnsxx*pSd$oc$ z7NrVWs$uiTM_kb+15N|OxT@6ZpeP5V>P9j$1MF3q z088v3un4 zCz3BG5k2;x6N{p+HT89^vUu?0gTFIrW-~DOst&MuujcXOQcyfKn4^({wBPhQkv2A$ z-C%Df#1%R=%A=nGMi0u0)FX-*Ry{uI(Xa3pzX^06xSVqy+)yZXlxmb<*2U}gs^lTZ zkPduh0=mX2v`U-eEJUvD{$hY{GWKmsw|T}Lo!M(KyLy}?Fg)!-M@ISLI-B`bxUR&6 zuR>02X@J(~NT|(T%HHFOUyze1mGaM>{%fWo2)6_eL!byaV8i1|zdyb7VsL8;To4qE z^GJty+(SanF0(}$BRfKaNE6R%IyZ3#YBuuqX&C(Zs%&q3vJYC6nUKR1iX!>%#?7EG zNpLf#k$nLyyG}Abuua6T^CDkH;nf>W*15h|2pB2GiAS8cqPaKn7cVlj5RNC4MM^yXV?zs(O7C$LbqdDRS=Vej}hzQnj`Ykq9Du(k1A$ zpu=zXGv9Y{&vt`pKF~Ru%FT<-$h#fFAr)pK4xKV+*}gK%6ozR~h-=N_7=GicJLxsB z${<>m%I1TTy<%jJ+*$<15De)jIOCz^dk^xblHid;8 z@kkS^#vyZ%3ze@gK7RJ>WA61YGg#(m<5UW)v2!pSA=QO^BLpm;mOJAsredd5VCV%L zCfNy4!+_WrX(DecaSzj~(uX-KyOAei^?4S$hoIzj&LDnxXiz+!7=M(~z^+KAv|ir3 zK|rNEK$NoMR;n|7af}ARZZjHEWLk`IG2A=^HxOYq+3&h)^J6pjyd)Fc^LCyoN%HtS zmWv0+qcX>ViYdjte_ole*cYq;);Q`512#tLCAU7)gW&zzn>+Y(Iyi$7?{1-qe2o9glwOZs=;eC}j&r^h`@fSh*_mwDY{ z87SQ?Ig2melR*W%aKE5~N}+Lh7G-6#?&=WYY%etm!wJQ7JaW1DfQc56v1P7iWS#X)O>6-JDp}S$pZDk{{koA~o_mgWz0>{u9GDkCAi5`MWtwwo1it{3YKTAnT21Y>9Y35nLtv0&*cn4LT+-_+A5wTk51H#am#BU9P7)rqs$_@K@!~&I$Lkwv7;+L>6T;t z8Y;MWTQ`zg55=ousU`U(-Hdv$U7nj~sEpDPA`)C@J1$O5OT%{vLXs<|Dns7e&;mY06ZOm#=(A51!opE;2YNs$k1@#@#~gi{gy znhEOLh}D-$0fBf;&fx6NNTwTUGCJV;^V6(2^7TfBorz=TqDl7AbhCpFyT1F`R9Q}8 z)>zzUJcZbWj?J$o3sjGGa*XVCJtULxxw}-dG8PXjU=xP%bB06p?b66e+2|V#+j-r< z*9gIm{WO&ET>q2MFWilo1=7DIabC!PA4!p)cW;h!EG8VMO<;($s3?|vPrtHJS#Jf1 zcJ|RUc&6}f(OtSuEf{GywwSG^g=R(f4VbbuiZfPoq<`_(=Lmi!fQg6tElG0aFx?Id z_I?-wMECN6?C^2=_NfCO_r_7hPQSmsPmal3LuJe!MDBP)XwP*x$=oNyM-LYtpW<}r zFF3OA(%p-fU~BJ3Y=WsFaZasy;o@TEc8L}ma5BTqUQmB^0g2%)1Y)WceXILh>~c@% z^fw07bEx;xlN6z|()hTz&Pwtze-mWFk#%jl^gcxB68F)nD&{^@e>Q3Sn?_&SXNj8$TL0TXf85 z2vPi&7Xi0C*YDJ!G>TCxHBdnYM(nO8`Ix}qE6zb8c>M=bU(sP<%8pl?_RWsVzKN(9 z&Q<8JIAAkL;g*Z!@`eZ8{0UnS>YLqh7X$}!ZjK>KUjoc_@;IfrLKz=ySIl;p9^>YL z&mw9=6m4!4Hn9nRNRyR?HOS;~lFEgbx=Uk^)Yi#cUQp$-Z^?YlZVOD?%L3#ghEcBScZD`sq@CR4Z<=U0>lvfn~y(n62kMbHf-D^Nof&ron%_ z1MMCfv!YnUP~NWoRv06V9W5pWebok}uA zKD&^iX#8HW)w>nOa5MO&p#eD}_dWa`3JB}qSbGaY5eZXf^yL@U8wV+ViZmPe5 zWc;i4>j>_AP=sq;cz%u2L2bERdUoc1La*CR(Lk=rn@aJKjXaawrT_B5@iV&`wiDaE z_r{|Q;8d#y2iKaa0f*0Sv<}M`^<&g|r~mf+M6A>0aA(25e_LNsC#U!QbMB`fRO9r+&=j}uzt#{;Xs@AGk|bzWG$yht9!)b>c5nt+ax`DMn5${p8t z9$6-VQLG_ttR7U&$gk>D$xE_JoF4Pw`I_H(V8%GEgTaZ0@kpGLDJX@`p>s1;K0rcG z@i#@PV50u|frgO@BrO?B>rTQX{#sSaB;C&)*LfT7sYA`3V-s7^n4~VK=aW>z65YQ#E>WJqbnck|m;+BaEC?UYlI1 zscp{f&yW^K6GImB&gk!hlmhuGVA-}8Zrz|@9xVKm%sto?r2~yG`sYq0nDm(FJzHb) z!8{CQZa@|{3e_CnJp;=I8SwW>MM+*u-FE@j{oAf^Obfic!t&w11_H`qum%3!k0`Qa zzBBRU4N;pAzX6co73mXQOd9ApjId>@GbzGf#Nj922{+nTf&6!V*;O*p6dOYl$a?0+ zwsD(y)f0@>YaOyh)v`e;*}K9Fd&HwiPa5@%|J>!=ookYf*H$QBT1q9nMW%p;!#>W= z27+4E@^%FiZsqmien zAKssmP4VSiM%?!Rddn&U?Jkb*w%bHd&K|}E5U$+0TbnEf%%vhX}!vU3x668Yn60}GVizuBA$Z2nLer^n)l%z}7sh<#j=n8Yj4`&evD4W$tD zT9!1>NY*IJ%0ziwq?ZcX#uhh@!-~O|BEsE+tn(&iC1M~)*M_JS6p$h7{~nt>l$LdZ zuBYOjL1?C}tc&Eze~KM$_udM{ZnAqoKgAi3O~D|m<*#iv>5 zQhXZ)#E`NI4!z`?3(#Nk}jIkyF@h<(5Dy( zo>5Q1+Q zFi*%%qUTn4dYxcO!iUlA5UPrQysw^@L1IaE<4R8?VzAgaRPH(=gmt%@w_V`2UEjdJ z-U%@d0l7_Qx{>$&C2@Yj_W(okv_I);yutrdQ#S5SQRIawqGVi**(wuPY@Se;FHZ@U z1JMe??-;5{e7>8Xnz#>pvD^Ol|pcB?z6>t1yceLD3P2;*(nIXYR5i~0N2GderU+<5a&#Cu(BqjW6j zwTcF=|I%IpV4K<}A5_2>OdP6mFip`DQfSx%Eg}P|^Q+A7?t_5;05IdpD+F?c0*wp$2f-F>MiSfy1dzn(x$zR%?01l|$HxjvQ9 z!F7u9RM)YJ-OmL%f?CW@uu#6aQ&A^loVJP^xr)S?A>|1u?HBtU3=NC5uSX>11)omi z2iwJAR6MI4ur^!E!Mr|D0IOYy3tDSx8#0$4Mlp!+B!8${sHoc~Ui5kJZUTkj+lHO3 zM-Ih54!h=Z6z&wRM}06e4gIfs2#E#V>llL=`(5zY5aHwt#m%q9?=(6pp1Pis9goX* zKb8}ZvPU0fHHFJDiEqIO1&}MO`(@QfV zKXqvcPoZXcXNiia@A)vC&ZG*ex!dBQ-JcC#pB=snJ>{1l%kEkkJgaVBR7P zn=&Z|o{)!8r(th}c(>JQNf+u*Wgxhj#V!0Cbfay%M1mB2XmZ}b=B&Z|89z_ez9ryT zT4LlN$@~jhZASFf*W7vVQCW+mjm3M@RkQUo8Gy8!M##JQlQl}m4eX1tMYsZl%pZqE zW2jj<5l*4LOx7^%WQX|UC_LNMhZ~9>lZy?%Z8~+I&(sND@aC?CN%+{Guz}wJNR;c) z6EQ~=PUIWxi(pnw(LplG(1y>&kqo4Zo(;G`(AVGEL*JnpAt?4Ubo z^Z$tQK->Kz|MAqnOX|~7WKr{!qTb`o$coZlS59c4IOb(ci+|&n!J+kAS!RmtXI4#a24nB~f;`gzc(ulv)IS;a|kc&wp zxx(@feY!vFb8g%JltTLl$tF@mpo10KtF{21+KTD$quM-E;fjyl{oo>jFu%GhuH=I} z)bmjr1_CD?Sc*_bv1JD5OtPz=Q`=R3BO_AdqPY0{(278oyC?nf%ms>h8H-zkqoK)X zt;~d$Z9dm*0(wY<2+U!IaK$P)%FXabX9o1mqTY)MkCi|fF> z?Xk{2$}<*TM%HyC3a=L=&WvL!@oy6jGm=xVrSY>f zTT|a4#d+1?60-41o9BupX4v$9Uk zR*0V_JI@gdh_!S2v!^*$_3NX1l^sf$Fn7X&xS!01>A(~L0}6kT%QRB?zhSSJ_GMRQ zUU@cS8U+D)AL(0s|B8#y*j>OBoVTt3Q zf;`RqwJLfUPLE$%ZgPjQxF6_l8?PzdD=fc4z8wGR1+VN61J3(!9w2E-f9DX`iXdRwZa$u;XL>=X4qOcY_N52m3CNi1(^>QyQ-cZj z?0rnBn`$u-I`g9;3?P*?3I;V``*4U4(W**U^}kb01$Rdhl*2WE6mE+ghS3ucmd&gp z+uaq#w*itDeFmbF@NvGT8p=GiV%a>REL|)ju?NTZjQ>|XoX*muQdn1ek>)!9LsjPT zl#fqMaxxX**n10JosKDG84oeRg390I*l0lYx**!0dtfSVR4i_l+{vFDN)Xc^5FZDh zXl7D}-jfY}NNHP8l&Z~2<1e_SpZR@e-}9zOG+-6hnQc_Qv?)!ouj#jiHD?N;6Xq~` zpm&BVB9tJyB)K@=0;u6M^gASz#KO-BO|+ozU8w6fj*arXC3OElA@TT7SU7Cs2;u1H zAw=%vcDOi>JCOE9yY*v%UG2YnB-Jode?}o(BH1zvYdM)m%AiTIAr)K%ZIPksXx}(XfPyt>%QWBpNv(!}DALc2zG1GPbL#BgIX)Z)4Lm-ZM;q7F>%_^Xftqd#|sJg4ud&QT&G*Ju){oXFKVQEbX#fM;=@ zn8fqf7Y~L*7WXIH;Qnu~*XkEO@_oh9z@U|#6iiGfWh&SRj_w5QYrU5KFl6y(z;&D@ z$kWcb#r#aJ=hX~JI`b`0j@X0$DQ{n(GG8~!7ud3hx?=N^PnZY3@dl`o7eLKH$jqc!bj-W#h})Z+k59`c z)K_Du&6*!?gYAb5X&MY-96?WxZfOhTLtNYor-KrdU4IN>A_I+bXXLK{C@+;2X}(2Qv|>0 z|GB+4MCzhn{yEg>44NX3INS|S!10~^>A@_%x1|E-6#P@qy3LR@Nwx7xmC9^90QE1E zC9zr=Q%n{)W+W~L0c1?aaxGh23!0<3v@_6B4 zMDC@$1++&5xucu1*(i9a&MSTAs?L^I zT>}DYHC>}8kOXDBB7&gYy6~v1{+z~VE2dA`~efLK3&n5EB+?qLloS)u(tZ}jdG6->vgH&~&)TswE2TZgBp^F2#DC)IJgGWXY*mc$x#PF z>BT+%5HgsK`r9CHS?*)f38()ddK%abkK}E3JC98FDqd;AgPZU^1==M$$%||{@avjU z3KWdL;B>)`da-NTT>_5$X7dDZjgrRywA4s}H4q`vR`*mPelYCC*YXy#9~$AWCkH+@ z0cCDmiw0GZ=X(g7!+`6(23mmuG2;cRWDOkc5v#%2Bu-|NPR|uy;Deni5tma2F9Aw} zVpDe518b7bldLs_zQn=$QG5X$K;7Zr++oas;X;)cMoq0i(|@dEeJk_7`<3i9GI(9p zcj(d;8nkzl^y=Tdh;^3*ePl+sD|XPlt@*S(vk{l_e}1V6&6eKE4bgUb;9BuIA<43V z=~M;Qufe}+o#wb$P=^mrTA>eDgCy7RGvo6a&?mfN+lj!}M-rQdmmPqt!z`8qVE0rj zAWw5b4)jak!n~7Me2NFN6e`1+isYf(s(PAY#B0X@h@bUiG&(igE9VL>kwvt+6!k7A zCh!rxE+;K^yjMcdr9G1->9x!iQQ9^lFgd6rBAk=!$p!!e0Fs@mCU-%6O@wjA--Xid z%8GQ~Iy=t)iM)c$Er{<$ovA#L_LgZZ(C^9>H2zwN4C|m6>L}3GL$57^^S4w`3kod! zNU#Cv?m(L$(AxgEJd?Y1mRRPdGJI11FmH@Wr61|`Eu+Hqyf2}_eZB4<1P1N3pxeWD z$6pDNXx%AXvwnfc6A^!v1rX#+9D1QP;NYMt)Xfd7v@Z?J9z0iKxhT&Rnh2P`4mgcM zZPEJ! znR|wCEGD4chvP4a;;rfZ?-jG-+w@c)s4#+YvwEK+iVE3&TGC8ON(F%%tuiGux?#xI zi|L3;hK*X_Hgd{`wqgnw0Q4u|5os9n{=6w_Y_HEwQFb>^c?Ge#P{E3wu3i=Tw{b5BP>|AG&q>?!|L#fBQSXcu27(z z4IbzHc8d%fi>@3pA0NTPyN@8}!visbsKS?OV4rBl8Os3I^~YG*Lc!d$pmlO0Y|~22 zXw>|=;YJy7R+s2NFQ2Vu%rn$-ktvqui&2;T?`$htDy`GYTeF(KUOJ5$k9>MWCSqwEZTlZ%x2g~F2eO}?&8s9278UEPDC;* z!NTH}i|XnCpmx}Q@lp`J9?eJ%T75aPLA?0|{f2bXn|a2HU>z^lPzzbgJ(@6R?R60B z!~iB<-?OyRrOMk8uk^}+!`QsmR0Q(@!>PQ}a{+KvW8Kq*LTK1F0M?ia_@oF3Cokqe zq%Jyiy8C`3jkmgg?p*6my|rMmk3#E2K~_O-EviV;d2`>@I3d)VRD+}^n5Lr|aZuyc zzCm6|MDJn-VJSd_tunn9^dOc5pXNV@Mi)~5XXuVh7&=uJfsWTr(L^bd8B)MW?o>k7 zZ6Cx!L%g^vxo-m!r4RwU6)KBLW$p+?CNc$VAXw46Avn-O{_hA}@c~5i42hzl_NZA^ z77de?ad8Y(o4GvrA-;}&tZ5XEn9B2ZVQLl-s(e@qnqZh<_NI2~62>c1PsOMA7M6zlUv?qn_FONxl+$ zhhjT4`7#@3MeNip4uh0g`l+&1k(TYXKysz$n^-5?vi!rj5vT{C0W z^yfN35m^gyZakr327CC1jgmm1AZEL?KMp$J9*jZzbX_Q+9--0mKCStR0O(a*|Eoz$ zTj$nV>R;13;UnrspClzr&pxmqi2k=NjFXVp{l>FM9Qq?5V^PcqC@1A;>$5%64>uKb z$lYkuGou0hoKu7u77Eg4cqPqWiSM)o#~gntEfsKh#V{l2oo&9}s-81Z`bp}$0*it) z_uYiIFvffQy+WQPPT-VA1H5o72`zw+z(S7AT^#|QfbRWfn_KA)+9J5D#jIwr0Jm;Nu~3!$4ZiGDb%<+^>W!1dxGK7eEU{1&atJB;9Y3VJ7+yoLRj z+d+m5K8DwybntgVCJYd8R@lrp?a9RY#4rt&DN;P081it8C&(A0yIw>~r>Rc%O&P-n z0-53hTAPzX0cc?7lBzf3#6> zh~$1tNMDDk7=LW1uHSc}A6sXJg(uVkZ?FxOX?T+<`!@y{*R$B6I0b7c>D>a~kfm=a zq-$W=c^0~Dl{930K|J_jv*Eyv*yfTWf@*=dysVV~2LH+9gz#C-AE8kwCiU4RHFf~jrD$W_#jaiEG`j+-$m&@-k z9)U2fi|31+3R)5^`w}NeGJV|y=;J4Ca}|F_ya8r;U9~;OoJvKRfCNUbZRj3L$zL(L zjJ+gPp^FoqC;``|o~RNg&6bkX*~e_{kd7;!sZO=cW#Wq*3AjG^T4I?zB*P>#m&T4g zT-4rwa+GPa%)Q3qZEw4o7hp$d##R^>H{~{(P?QeLx0)fql5dWkJMmJCkR=`kMGqAW zhUEhcykvq^tH^&c>|;%bsR1LHO{^?8JIG;!HyL20W6O0}zMyhg0npF$yu#dH~h97=V#(kcJhW2-~4%8I3Fc&su;#3L66G zy~&U6a0Wl=-+C~4vWjIisF=&R?Vp|c#9IE52^uREu57n$TF5v0|0{$i!QmU@G~)`| zxC>zM<;HE<^I5L{i$f_bD~pR@oc2tRUs%Ji6dgwqhxGs2t&CN^Int{%h<#Ea{Qj&_ zHs*y&!?0Td4=DYWDB!q>6Ej~du4CuXP2?bo0PJfaEQp5`oI6ldkWTXi#o(rc%pB#9 zZUoQngJ@!X&z^-5L1cq=nuE#~r~b}@i=f&Ir>880EDa}4=N{|eGOigjba+-|#OyWh z*HdkwE0B2AgVnZ~`}%u?z@QMsccXsy7(ts8FBtIDg}QFsX-Uv@*D_>?m#R-z5LV&< zxZC!bheP`p-?#??IVcTLSV=SX+xHFc!PFudU-rFnd*#3X&7xm8=XBg(F!uRYWUBOVH{J9gn$A;4gIm+u-SH3E?St=# z(m@Ix?AtqG*P(xnr01&S>~ZwgbGVQ`FS?U!D<6)Q{d%L=GO?ACh8^IMlrI4Dp^lq#diC^CIrcVL@LggM;68Y<>Gqu*^aRoP916m zyhk^Bk6XN=H*qYK`Gn>YK6?z8Ntxh7v#fs4E244K!}+uceluBcs(&t~4G3 z#7GK`DX2%?S?x|0NOdBikyh0OT2Dw5!wkVCwHBY{<;9%oYAj=@Dz}+95Mv%o`w%VU z^%95>({0w2WxPV6)%l;(SgyuA+S#VVa}wRY0xB3`^b~6Flvt8?>jS=s@#P?Xj7iPR zk-M$wm1%}B{1Mb>CfKo0t~z&j)o|*SFq*9pU=fH7ysk5=_$fT9DXzY=$N;DQf#f$k z-N{vdQ6aW7Is?Z#p(TfbUb%!iVH+xys2<)l%&cXo))m1sR_#kJxBkqTFm3-6hv=*A zlHK$2x!FYHiyuzdo%{hqg#1aHyz&s1c|LYy_8^qW!jS$JjbX~j8otSoTXNX&`R|ms z5=Q^D-+0BNZqVq|1QIa4EtKND3k6|gXS(WZSnqZX47==(rqs1MD<^-fGSnCy1++MC z3F_1uo+C_*#9OgEnyRzol>y31VfQGzBZ64C0u8V>kpq$y4_IVx7huSAYc*^McDjQh zMN6FF>y{ajFdd6B4ZWwtUM$EYp{}(eTZXp{XaMhzV;hdYeClHbeckJ9m5*QGB?`R+VGA|qZp=4Kr5>})~BMhG&?l63_S(Pub z@fi9^)xYmekxwRa$|3xn7-GengYjyo(bm8W3|xwX2_r6DcUYCCF;y&<_9nF3S?Q5A z#C2dMc-P}R98(c*g{$(Z`)r8!W~e)#)-(@VGXd8wreh|zO~x@__6PdH1bXX!#!e-u zkD)peHSNse>Uf}@PuPumwD~n;fs4vUpjCHXqf*OI`KKQ8y}nEvytowmaYT0hH`3y! z-4Gd3)FkuD_ezoPQ=`UnW8gEv_DWZc30ee60JBgDc}WPQSbr4Do%)a4*=_kmOv-Z7 zRVx`(=s|4c5T!O`aQC$UF7_P?>+O=;VY!AGPfQFy!H=8oUQO&NVu!xe`ql9K96K)5 zW(EV^hl8TVsF%xd!l103?YQxmt5OjQJ8jZJg`v`|u;F*8+q+NpIuDA#xoCKXzDFEN zW^XMBrtV@4wMH!-;bj)}|7P{7LxWhs~sp}4HdJy zsK^16O5!mH_&Fy}%KjjNg(X~H{#Xa>1iHMmIPY_|_!X~uM$`s$%Gg$_8^37V&XIPE zlL8>?{Xd7`&Jw7IQBxHMA zBgP>iPL8D5v<{Ie2XssB_vhp)`b!ok7r1--QaQQOWITxoQqYXNbb1O`@ZJ)u%%iZrGXS(L3fDiSBg)bZTA6{(VC?>@= z!5Jnj#Pm*ii}1KpZByU0yC@AN_7m_rt8NWFK?GOKLV#Z_^xTcDR=H#~~ zwFx$>F8Cm_e#u0O_Uxo{f6iWYsXuNRE&6_DaSr(w_8`j=v!w`E7-&+75`)u-=6&l_ zM&gaM7v6V}V>|u@rSGCkmgya?@*HV>+hD`S6w|TlBiL;$7mU0MRkSIX0p@fJH;F9f z6c1Q;B;Z4QZT>W{YL4t?>VMB`X(UVo!hAb| zdUV7S9b>ufwxM%T)Z+Gprpi#KA3c7^_QKH~jK|}*L&p9UM=Cq*g=ns? zu@{}7%dE;+bHDfvF6J9%ma6h@Ln{wc!}%Nd$lJfoO&4Wp)NE9!NRA!d^1{|Bo}42Q zIq-@w!o=*wnUpt^U6c)$ePHhqd0O~MRbedI_a)16@658ZGuM$pKn59{4+S^1;(C*7 z<^`XsKRuwK5z>N3o+>qtE4)jQZmvTc_#*7|>aE+i$Y^$24(a6HnV20Js|Pt7jDJXZWq#Ra=n``w0_N)r|nEnnvcJewDbsjRk_ z!x7@=x$jeMIA?>>*2kb8$Ns3Uc2-kBw#T?BGQAFxBMDNBdA3vH_+RbLMNcRpMPNzl!MkQLoe1OvTO^p~N z@FV0;>F6`a$c`PGTK>A0krI4R;AOSP`s_X(q^c4JX4PsuM65mRyBrXRn?inaoKkB~ zUH<(%$TgFBDimfTM(yyL()(xOh`Q6LrkeN1Sa4O*!kk@0=S;6-3FB(Vt#dQ$X_J&J zdm)I2AFVturvmt%y;BK}Mirv}Ge|@q?^e0JUyh=!l{ylG+~#+8OgKMs9EiY$!2Gp) zp0UKN%<9bBZrLZ>`e>|7RR_ouGIqhO*C-iOP&1Hj~F z?UVl@=LjyR!+gHW4O4@%SqHn{Kjy7U-UZE`kixBhWic6x#cvOSpEgQ^@yMZkH@?i1 zw3Ks>#_RU|hR#vTAi3iC-0s`AG-^rva9~>sB>wn*I0v_Zq(p%B!+}>M9n}l$JDq9_ z-u6{g!16v>d#ihYYFB;@^pF&+8BtLFI$5aU2WUhwNcq|e{r3rV41Ja@}GKL(sALGlkpCZuw@#C zV{d~0@wsb17fNw*twL8)eosdWGJ^qSfyQ(INkF#01y}##-FBuk_}sPH>SQ}T&bBu| zC?0qE0*!*HA*)(2x6#Yu)EAfnE-Ej6lU2i17J;+PkN3TcYfBC`{dZ2zTj!kN1B({g z4LXIL<$jk0>uDQVo3YCx^7%2Aj&)p6TrM>plhDS{feZjiK(@bBbljgCxEucxvo>?I za==0BD$YXqKe?zUE#O~1ToZZT56#uA%s39*EV7($n5~_~W|uVA=c{9Y++Zt0ME@VP zkX1s*p+<_BPp1IN>r(G>?UFTE`!1w0sutj)U<}Q0LEj6V|GCG%m?q3JIN39Cyc4g^ zFg(Q@<;&juFI6hQ1VaD4hXz(or-SPlbLEnWw#fdXp!>eisEHX-N*m`TnpCrmiL0M5HZo7P}+<>Lw zqio!*XqVCkY5kCbEBX!=R8G5)61zRe z=UQ%-D(=aeCp(eZ4d1yRPyXUVPb8bX%s?uB`=WYyT~QwUZy|5cks%=}RF|?oJR=Rw z@`Iq7`?*SMC8P?|W9@56^p)VB--4x5r#tzTp_og8Tg9T#a;TrdvqYM%8*xJjOlL^M zUn?U#qmv<`ZkG$#C#S$Iv=%iXrT~_~yhvoZ190u{F(LlPcHtghAo* ze}xKJ?OrT!EcgXdsmFJIXq7e=-qhUU#CMTd_W3q^wyT8ny5n1OA$Jj3gen~|E~T|O zv6{TEwJw6(-Z6F=g-4HgB?4M~Bhjo@&M_Yl*-5vGXtN5!s^+S@;G*Ptj8FxOzOQ^-s3J2BY@c}eU z;Z{JDWq9{lBcni==i}l$yjWoKsvwY622J}kWF>Upn63A+yosE9pk)o4$n5cb752qi zgSjl}Ws|doHSmNYrls2w^l{xhC8nMA4112x#YTcCXY9pIV3-+v4?DDE50`(BGtS z)o+GBExEM)fy8UA(0TbLQLVM+K7j}w_eJma^#Xnndy!3Bsg4pd38lkB;ksQ-?t$@K zM~vP3-U%U6xk9Xq-k%)TduHO-oCp^)uTc^P*Q2>-RGs7%Tmy2q=U$%#4HYpq?E$-f zFPA2I!4F`5zPWv1MfGR9vZF} z74Ddz8qjDwW~Wo$Kbz>X_rqJ2HBYH!@#$lg6(|SK+X|+z(TqRqU92opHjCQ2k+$o$ zAL?|-{KI79++;N#QW?#r)O_X?V7m7wZw`DleFYh`RzhHw~de6Sy2;mMi={9Dp1X;L4Odx74n!wwG$+b;lXBq1;f;vX$d|UZdRNwZ<$hUAa(eKLPSB9Xo z_|+S}e!xA_!oI1>6h!F4?x3C>Ayq|HfO8!PjF%z{>z5fiHCZ-Y!~_y5Yz6J-Q(chd5l%{lmLF zr2PQ|eU=bEutm|CB8hmRZS9zqR1geY*Y#5SX9W`fW`YYVZ$Q3o4tjTTbbBG;GZ&Kt z>UoQ(abF+AYWa4g99&!{kB}#bTYX|>m3_zT{Y9!eO0Pp3e5*+S1I!<4ots^O$>s$w zX3tfrF_;J)d%5U91eJeH`x#aofo%qc;v8^1P>we4o$msr@;!Snn7*OMS$>4wh>jf# z0WJr2GK-1VUzxDD&uDP@@zjR^B84q;bi_;x=2tt>9hrts-DVTstv<+XPkl@ra7bRM zq8Bjs*9|t*+wWxp7HVAs1G)oMgeO~)BDpZ?o7L|^T~*ds_iK)xowwB1J$leLH9i*8 zT73g$=KxL1$4@%y)X3)lQz5M;?U6%FwV#`#Ikd%H2Kj zY~=`K3|p5Cz>BHK8a!-Qff60B|6CMz*K#Dk{2w?0Cb-(aF-VzDDPD^mZ3s2dyW9`L zSn?TUJlh--nb8dIW8fE}#4q(8G230ooaOaZ-klo8tv=TfiGH?D1i!;qGPRc~e8q#Q zA%h0;mvQ?NU)BPH3ASREPgG&lehzUn)5=%r*JQNDxiN>cJtQ(=)nRIyHm`5z&fHth z(sVh9O7;rtxkNEt3z#L&Px0~7vCSBf^#b&`IL%#npjc+!R}2-z4MW$l6M2EX9OWkQ z4*t#6K{{1VtgTT<5tvyG-8qxph8)|_bPhuTc z7`=I8*EWN;I|S#wak2-nD^+TY9!-?&Y%<(wW878*b5dXXxxZw8x@FiSa)dD7m;Rxo zg-OV7a)%U5grS%>9>p|WNtMFLp>Mi>t40`9@(dR!g_}JrbM-tb?~No-QvXakP~4QC zu^7cz`Q5PLK3_uQrf)n6%ObPY*N%>OmUIqDblVM)Wu0&|Iinzt4MDc9?1(1-&J!$- z4%$f*)ZCsPGq=N4{+ZSIU-^hzJu8RC3WPK#E%vd&RT@4r_dp%|d|QEO*&&2-vgf*tER<<&XB!pl1kYktDUp4Q@F zAv9I&TjRq(tm}_huultY_(pXQNZ{wX{X{>qE=B(ONp?dWtwxjkin|rBYsPVM6=n~} z-y~D>h|C8!I&XPfmXO5K0wscljDl|@UJE_CndQI}*2$j|D{tfE)aO6k>PIxGZBMP|q`aqStG$qpgN9O+cb@ztNi=BV75) zgHR6Jx@Juj%4-vfvk#gqn1jPgh!n|`UB3jLXvB?6}L+f z{i@92(wh?xk`k`m4vo)zz-yI7fPQ0%0hqsawwF&UIm2BbA9$x)GKaExVcRVHGDez) zao&x=c%69TAt@E1)ioMT3uo$$9F)37od|M&#laBkmVTtT5LYxSsb7=L0W9IyP2H-YOdUBFupKWJ^3MqWT$491-nUC{{6Q(Wo>Ctd{)Yz)U z-VlqmUxco@pJbOIBsn!DS~7{T;zlVMc;Vg>R{emz5fhJY}$7V5OoD%I94 z?pqInzUR}t?f>Xorl0Y5ulz>9TPtI@#C68wudadKp5BtERN}Fi*)7n9b0X}M zB79+#NpnC(5bf#gd)zl+Oue;Vy~DcPg>52#sTP!i01#*U$ur70AG*fU$9*(ETqhoT4dU`U8J^P1#Ex(&G8soN?BX}X=#1D8WR;vlDV zfq;dfRY|MAP8PjqfbsDXUwQQA9*BT|?8a)`^0&>neXPMZF6JE?koBP^d3s;;t94hL zwEf)*Lv~66sHBB`!weQS#Y0^@VMEL5x8G1`E zJe@paBh?d=xSDK^$6`+*UI?p4##(6KnGJ%H7;c;I+cekyv4YybV{m?;ySesCRO~bT z?30Pq(X2ge`Pgm%T1~>;PTB+R`78tj`N@v0H_o(!-BUoeb^z=r_kJRMfT9B^Zk6|E z1G1RHr7lf$x=LFq*|;{LrJwZ58f1YqIw`LS0uvc|E|Oo&IbC{^w9WQfc9#g2*qi zBLJm(@!0EXWW(_$Pjgmsn$W>{OSd*W4(ah~*#2@?4Fd+Sr3L7V3E)>}s zTfq7NwekyY3+w5Gn?;TI8-Ozq?iO#&KZt=jmVt%b$`%bXLHdbxlQ?o|tO zb)nUCC<#Y=n?JWoL7kBv9CaUdv>j1nF%;2RnDu=2zTeV>v-hFqi|tvZTxiy`#-VQ% z@*M4mRLwh@(7jtC8lVs>tkfK^8WW?7)z2h8v}kF9@}ZhN&6e#7ezGWB_Uh<`B`H>Q zd3!+^-9MeW2TEMxP6V>jg}eKPj0J@BefWE(bLWBX;a54XQi2xJbFLF7!v@w0Y7I7@ zH2g<46Xh7zjHOxbX_Oaovm%eCDIx=6*mf~^5zY`0RIJJrjJMwM6(3((8@(_5{GBN* zcD|X3hsN0#nWAPkx4EpsZ8hl4@BkhsuI^d5r#4sxU;QD6{t^#!0fpfkRpaYXMf*!H zV2-dtUmb4hv7L6<6b;J6%F+le@uyh+HaYSp;hv`CPF{Cae%vUXo!d(Mp8134sO&GM z!AuIZc6o@6b9qS}<%Hil66HDhdNSP@77pziFQOLT_Ii(V(6-W^^KzTH7md$daNf&X z14zu`#pUw}G?TOJ|v3HJVVnY8Pcyu3_E?kRgScgikq(qC) z5Sp&j2~^npT@{nwiHS0o7kz8BNZFvC0FlBqgBUH%sCI31Pd6#fqt4lZSRJnI-T!iH z#uFjQo-V=pbm*Kz%=BZ$=57M4V8A`lXGV<9?NZD4PL42XEr1wkYMMbIRKe<$9K8i? zO?ufTK%6}TQ(uXa1Tl3GfFD>MX-vJ7I0E~E8^y+?EnbY^0>h>;V(C8CCGvTl>D;LP zQMFPU0K_@c)cz7OeuD!*ksSujh|NCHqcEodyF_4=3i7>?^+}A4(`^iFhX9lBEaDYV zuh$H6B`O;t2-e~+hj!q``fSh$ljpZf=(JOLs&_?QbLWThaYlG?OH_yDju>lUT|(k} z7s(=KbGXDS6TU8Vu%(P8bgAGeBDpQlYefboS%F=$^^xbIR1w@nQ&3=EbwDutcBn@# z1|2Y0Iua&OP1Rs$?zVKWCK27@VVOuf(<;vS1J&p-b^jW)BM?OePzvZ#Zq|J6^s+6f z4h3p_v~z%hJP*KsrL#d1=y+6;A*b)84bj)jeDw0g#H_|9yuat8Y`A3L*@49-{DAA6 zeu*}O9+C+SH^qt`w>t;{XY}Hgspv=#9bx*e=7R)>DOMbXUc+fmKDMvLFXU2I!#ilL z*|ur0VO%L2u0Z@WO+#{N=bs}^BcSpxPW60wZk_UD3)pD{`6oE_T0#A85Q^(CwHyPw z0HT{$w{Hh<)yxSS<16z-#~)$z+IoX8_cwchzU_SpLb6D9f=#7iB&zS^2(`aF`n-!O zHj$*J=o;kUb`kDj6e9BUOy`MBCu}tpnmWJHVa1MeYv2DKN`uHDC$~sQFtO2h0w7;h z#z}tBrtSx?d)feqs`~)qA}AdqP&Ci|bo(OfXl6{?_IxL3NmMLF5A&2C*1d((!=7 z>pK&d(&@2y)nS^5y4vWzPWIQH?zTH(IgPcqs>0g$?X(1%t!Z6{lPvZbA$h&zkE*FD zK;bJSvH^lX)a5z|505S@;3xk&xu*Idqw47B2@qi2B%fyY3-ODksQXCU3)(M?g}s>=GMh-KYp;k9K2vz4kU}37`5S_i^vi`LUcSo_^{1d9?G9T!!3O$ z1VarDAx|bnLLq$Rw8S}~FfW3wSYZgqy8WG5>gw^NQ1k7DX?`lulgg5`A&tx;%GK^o zjzOqq-MjG-@q(%Pdnh)5Sb~^b_Y(HtB!JG`_-M})j}Y>nPOflh8G9`i+;UD)sXzs6 znS=X2_Aq3Q#a=+J{Uqk_G3x{eb{wWHG+=p5LrcfGA5V<3${Z)$+1Gv3PoDk-jP)A4 zVzwtRM5M>67wb2u+SGLmTot54BjuT{$sPBDj$N7*(LSiD6|#ny8D`so;mt8h{pmO+ z?W`%r4rB`$a0y$}-PNIUxX5pcQ!ZqIDd53zmNV&uL0(QFVBbrdcnA{$!Bb^9PI0`S z{H;~ZOB}3Oj!CDNEAJmpLHn{WHb|F!XXocvZN>YBdLXrG^#`3w;B+r^Dtk`20iKmu zO?~&YA+!WFKCF4tO*f!?jYwpS7+E8a3D-vH$%la#I6C>Ik}j$9UnS8ioMyEM+ZJ3z zI^*$+$v-9}qw@?!JSOz_x%ZA1tZo1n!LBRKbodgoCMz}Agg!q=;AiHei-CzQ;xuSI zh_6S#-a;+evmbM5i$uTe)1*KL7_O>8Fl~Z$50AW_IRAG<$YAoGLM#sz7yJ7u<#7^- z)^^tsbYS#ozT7wJQ=D(~6hooo5X8zaL|s(cJJuHkEql|+4Q};w{t2hwhgDvq^*Nde z2>pq@6AWG^Zirrztyl2VAgJ!MYo;61LU=9(Z!{&(oLkNrD|XBM>WI6hki%Eh_)YU0 z3^Ll3lvR-l`dD_-?>;3sq1L>qF$PGnI6=fwr?etC>?Kuxa?hqoSdWqT;l3j<`&%A_ zqs0dh)1Miw@)!*ZVuteUHmK7;_iW7eNcLxG=KN7xy+3Mbc%Vk4q z0rl3Csikvwu$I;{JU`tVD^6%Xe2rN#k5o(pjLjB zNeMpZ&MwFaj>PIUw7~RAS`45j@&D%fVJ%6xTc=#via;lihB94h;qMx?2cv!Li&*Jb z0S;U^mSon&7ioYl6iH>-+CH#smIR0$j+_F~!-oF1ae1}$cS^%iM7Z3eyE*lLLdSOt zt{0@Q(80|IJkALrSga1BLqFgL&VtQZo-tZ8G{%nLF|O^?83YAJ%G>U@w!r&pUY#iL z#oH+cX5i$5$C&>`lm6E4RnGt_bpnV;Cp2LES(>uCg#*jr=Xim7))s`1^uLI86{9J? zuWuXnxf^SXzE|Ld4M=$PKNXTlvU(qHG8e>Ho5A!SPccO+Zj-4s+qI;^{I9|qQwo3I z>RSG)cyE`5g8JtEufyuc*DFD_Q*}6!Vkj)c*v#p_YH@;LS))nL_YyPPn%cX@bBM2{ z#f1@8QT$j3CRz{bkv1%|T&=4-iZh2Qj4H_uyXgCEu4l9!$Fe^^q%{s7eaEjp6ES}pXjSZLITWUNL$Y1>UVlc_*NJ1 z?jeiw-WGIO^zp~p;h8#V*~rAvB>>bMI&&$GOP7h{y9!RL>h7pgfK(54fRN{LLB(_P z2U_tGqbQ}_L!YUf=QdqJh43kA_6qQ}_Qq#sz+Cete)ZTy*ekhv&aFoExo8u|E040G z7N~!(3oHG~q>IV0mk>C%xln2b+3j@cJ<6SdCG*$Po8)qD zLH?yKxZ5*4kGXI9)@pby!Szh4a)fgW`tO{ssD&a&G!%oM)z1;Y8eztRz0AoB`=`TT z$I5z@RmdZ4!HAfOI#U9WZ!O@<&R?K44T%CaBVVXX8O(ujasfNgPORqRC6XO=07T-} z*U3Oaz_HXF^jp^ibYWGC2}hc@xxbon*HU~Pgu8U=t~*HMs=%B;?)>ng&MC)!xbw%T zfM$f205=D%Lg(vDM6Eb+ezyu?HNl0)-r62bl=D)L1exY%>W)aIzz*hk2}?Ylt87cf zeV;#e-Yl}Marle3h~7hxT0t~(#PV*SG*m_r+kR<@Z-cKkl%%ZAy3z@dW{ZUxou0cW z0x^YLl-WMu9cuGTB5@Gw6)H~MZw%%ID(hbd#vZT3EYN9-(uf(SoU0lqhrXM-n+y54$hcl(tJ_t&Bh;1~XW)ic2PP zGx}}3*CyDfo*KF>q*gDv=BDzS843=0dMi-MAW;u0?+w;=7UeG_m|kdp*>mwo9=-w< zjn_DfC@131w%pzQ^7Yvk1M=h<%R6`aD*bw-ggH641cLFFz_ET5MP06p4s{fTl}c7tDg*L+#0WmC-B4E11AU2N9d2pk;m252^Y~q3_ z@*uU|*+-?sqrnm#ZnI9nuE^eCRTr8LK1|^z@*SC{zRGePYedo>i=Ke7rwsQ7hI|+p zuAM}e`6FkS2_cX%x$ollyR41ta9xEv=)eJK&it8EXEF}pilZ|9ptJS`oVppmOl2fX z6_J>qV;Iu&E0^)^lpXQJVp3WJMv^+RNbB4G-1-+G#sp&57+)4HOp&(w!DcI!0i+}B z$J4r{vI3zA{|2~hE$<94gX6?hR1xm|Ci{15O_!*@#BU$ZzulXyLd%zGCFQ7!sKsny z>x4Kz-f4%jevBtIjo4RKP=F3+9&tcy{#0fUkr;Th8bQ4Fkb_b!GT*%E!-YKf3oxWZ zK!KxkX2O*Jb^deZcz#V7xTTnN$tDyqPepyx>t_G)6$?#e<1&O|t#Al{9~V4SLGUT+ za47+WsGa)sEHVWsc$2j`w(2uwv%cx9j_H%0J3`C^w%jk$EZ60Zf|@AnSn$tR78j)H zNXu^}7O2tA0N%#IX{PMEI>$KUqQa>vBXuaz^r1PS{4?V+iUZ!M3I6pX7wZ?VE1^Z{ zG~To_57B6+x>vp+?im4jnP**w&+(~TL8ty5J45JJy&$oKQc>*N@;#&oAvAC*Z(e9|2?oi>ZF+XMPykI$TeO#@x@@{6c z#Htk=%C?f~Io(^@6!Pz9;p?usHfut`*XgC9mRO#apOY6y(O>Jmah1PXIfD6**B$g< z>I6i2W`)5^Z-}ZAE;D|ZMnQI^p?H3IH8^bJ?gZ#{0eZS=R?FfO;Twn;DzIna<5#PL zgeY|bjkUoqJsGuJ?zknom{Xbhq=TlyZ(gnxj(w$Etm5*5WUa)3e(7GHVOo)pvv}WB z7_M@HO6J?Z+OQXG?*5vPUnU%rb&V!EfvD-~N z5x?}%pUi3qFHVNeSEj7r^j7{Nz!+u0zv&0cYXH-4~xWXY)<8jdd zdqXC0xg*Y_w0Km=o~|0>`g$ynP{-d2Eo1*=ZUQ8br0L>Vp}}8VgoET6EkF<7uC2l^ zOhNz0=-XvK7Kl*x|DK%c1>G>L8E<5T8RD41Yh&Qj;3>mWXF&j$ zF`u}Um4|%c?mzsv_AQnc&_PG!&cK5*CkV&FHi^m66+?O}co}AIm}6J5J9c|(9ZDf? z+*k|_`91(V!5V0J7VGA0jsdC0;9kFtL8Wj+8p~ydxa~sg=1eSrXcxjS?v`nwdg$7d8kW0aL<7NRfECV8Kb; zB$8(@&OB$iBx@g7=4C?w?)Vj`Bz$9_;{v!rO+X?AydxRbxc^CTig*TM;L|bXDT$Sq z^aFFm_Dr=8+H`;&Xcj%-^UkB}6hP|HQ-QYsG*K?p)6Hk1PONMH3`xCFKD_v&#Y;)H zfr3L=VebAao+3()n7rNnJ?nVQGarb51f6P0tVf@aPhi*p#;87do6kff#9|`T(C1iM zt!}yD)tW$Z8dQ_iKkKMGi5LHAKCb-K*&!(2Q-#!dy$D3b+4x6{<^8rKM%gd*&OI%! zyCG1=@zX(`fB+v$XVGBUZ%J=AawbKME%_Pp<^iD2siu4Q)`t=WYLvW%A((M**--1O zDB$A<0CEQ3rnNuumWcaZP-pLW;^wvy`*!;43dA@Dem~n8PHu zLTu7mkp|1ow{$Uxc)~x-*+Zt}btC|86WZj;A*hXr{=IZS!%T}!K}$H9Tw%ck;iWE+ zcPXfAs-;*xDr1%?BGIfgOK@pMgX~I`BnwIU;49rWjaH*=#g~}aaAfA{=K5cbmM`h; z0W>3Gkp}OoOXJI*=iB0ub;CbGTJb39uSc^_;m2Jl(Ehp0I{yW z5!#w_I~VfvD)v5B2!=fY7}5xg%Yg~mkS>G|wm#JnTbGWxsi4^n(?2MGVT~LXyPTg+ z?}=+paDVT3D6F(K_Oweb&Fi*+@awJ<%jQZ;Tr{2s_r3dK#m;(*r%dHPtqn35pmLqw z@Ve=-ZxRd{bXQcJ4mi5odw%GX%H9@{mgkiLRRW*I2mINzO8tUUB9G`>s*Yb6re5A~Gh8+(90POrI1{)B*^g zq0-RBK+GAIN?n@+9*Bm}wgoa+rpGIqUEFalx-tYm%Al@c^CD&5n2(mESSbQqRlKzH zH&cs-OHCNTv{^X5HBw3uApilMn|fFT=4A&{D(wFG?zE+^ToWs@jaEXW#jU02%k@h= zP$)pUnj6kWNCW0>Gmx<;$Lha>W zZAT(un|R}1Pr9*_li*o_aei&!uOZYnQgyCO9&6QCu!KfrdDf`3>b8^$hhWU{9ywb^ zn$kg&Tz}Jc2pW{RXTN67scJdUml(;nin{i&AgN^qmChcuVaOwy|CZZVJK7hDyj;c?z`B|}Gza{I*Ax8r*CB*& zu&yvpVNwAU6>w9D@u+&J@`EC>W3bX9$=xY-u@sB}kF=*vy4@TIfx-S*wf!GqqEh({ zO?RU#Uls8*xUje(mfni8k&_UNam(@G3;Nj3$k@Dx$nKc}ntP{gc5JF2G`ujWxD}jC zWlxJSzi#GOPgsuY>Zr}mhd;vv(8L+mr`}_59s1Higb6{mhW(rOBc`M=0J6a~8>|X+ zNZKxN&x=g)zr@5XR(rB;e=>1S5Xy_a1BevOI9Lxy?2X*fcV5V|U92mF9Aps!x!gIf zHAF%7vblGLh7ixfkPen>{8IT0BA(_Zs#%1_IS@L+VbcPa{D1{TofZ-R;Z{=$qqSvf zwKbHjiH6||#X>o{_yX6xM=)`jFH_X2x-vE)eVQ{g0tCLI2Vq8R4Q&FoQ~rh4Y6q*0 zqqW7;?7(05(t0Zw;EEpNVK336q|OcO8l&9z=E`Ku{sJO0x_6|$&VQm8!P;NZ(gA}n z6)QE+ZKYnD3*g1f5IYvau}Vcq#CgzM{>{Y0;v9#mt45L}KfSsUhhylbsQ zpgy`JJTZ^Abg(sv?Iw?c;xx^;(II4z6HHdZiUmk$RifRa6RhX}6}F<~&C10fpD=D) z6)9xs`8;g!2Yk zTb5H@zt7R_NOm$e#&0Jd>d*eSzYL^u9#%BVo%tQ|3>2(w)wR#65Ot<6#r)TC_(zcQ zBq@KHA_7EN5Xo8N0sWah{fkV?YNMXt@&O{;zbx^@*=&Xq_h?+xF3R*Jugr+D+w$<% zn(;Fv2jPO)1GqS%HU&OT&~vMSK6q4idV@XXqAGYe3k1#Xfoxqq4EnzmUf%fKz&pF| zw8)7O}fu_3Te1~Qcl5FZU=GBFylrV#;0cqWNvbZ9{&hnXNb%Yjcro@UX(497kj8v>R@%wa+8?}*ExF@CBv%T-t z#~%d<1zYaPylm?qqBt$l@jh`w4U*{e&sT}gXNMk!5$mHMZ)L*sWANsQc61y9=7dMM zZ4Z-b@W|gf29N^rM5l$XZGmK=i^}xFEi7}9m}>gQzEm$*djO{eYFjxH1}0<|*|2yF zm~5W~oZb z+HzX;c_Z1W_8>W&^5{^bV5e94=$uHjPI*xX8j}|ht0t~No|~1m5~rfx#+VV*LfI); z+ZdZ-0qOu=yFAs1dzuevwXFYD>)?*?GgP15jY0Xpm)jf&K@2>((Cb;V?nLMTWk%An z3dfg#Sswt;blzYF4u_Y!p2Mh*ML^{H&;B!Lmahi88!rEScx({g^Mnzu=9S89A>LDIdj$;OX_h8TC*G1Xi&ByAVwrliJ! zgge6SjG8*+_y6PD+cy@OYphXxT_e*gPT&bx!@}dORS)O_m!*4)#-x~`7@>=|Isde4 zGP>n)%o&XPA53GUET65hxkv6??0y7R7^Uw1pZGu#4?>NIys7l4K!Og+=^hq20qHM~ z!5;%Frl#w3s;guw8sG{LZsq6xWIC}qalw<8L zuL-JIR`g;SzAy0@F051^Chg?u>ETJHwKX)kg>uB@MINI)Qbv%bBZsM3z6qCWuh`0w zd750aOv6Y@g~;w_wq76}T(sPnv)DxW-K7_AufXsy#-U{Deo?+T6>G=lWm#^|Qoj4t z-yd##6!{y~mGgX0DkNGqg(P_BS>W*28?ZTLV~3}o!QBy+^T%x$^5xAHYcK{tg37nD zNP!hSKpqS6;TgCjz6!Fw>vjci{P4@??V7JymU|+{OpouMY9u|1C@@vkpf^y=9{GjT+1M@6oMy%MdjNr z13s#;>9U18*6W&sls=F6IGXMO8km9zW7w(%a0(NX>Hhl^<5yVT4pHB+$ zT13DcmW%Rjrg;Ug@FFQ<*_+WXw+&$zq zzw4a6AIGhg_m#*!jj1PRK=H5DbSWYTsaf6N+Lc{;9xpAEcT+D)KmOuNxpKa|rb#lo zQDjnZ63>XJ(?Hk0g}R%GFTg;w`!bcqmtM`<6VrLA&f|ERng4EE!@x=+Kc5#U7E2fs z5?{9@$C#)Mv|6bW3|{T^?vjC&J9QG}C(XtRt$kE*AnvMmF7_hjG3^GUmn%s%C7TZ` z(Nj!GSFV20tA03(5vvyV|6%Co>i&I0$33IvyYw^8(#tjhid2|a${Q|ymL`n%* z$af%XZ?@mncwcjQ`Z1%ZwQ*E?etmv150{I#%CI}^10TX4Ia^5ZNWRFY1ZY_g{a*kYZj7S086bNJ zK(YJc%o>(#kaS0G;gi|4ZB&{E-iXYj!vTIvRTsAgT+hA^?hAG7+iN`o5al>}V&lh( z5|nkEQ2t4m0os00+{O=Npt@tYq@csp%gTzzyRpvg1$pVqF5nU!N&Nz{3Ci{LqLT!s z!1Yye>gt$$*B{Uu<3qHStXi{{xx<|bStgqVcJtgKDLo|Nika825wTxFNKQ)(1M?q; z2V(nhb!6oPoeL0uHndAAr*5H&j3F(YE8qrB6Kv27BbTD*+!Ro-&o%Ew zKW90$vykgq5_M1(PH9!cJbh4RpsSD4PC*BTWwYF5UY!{NjUS?5a~8R+=wKr@OIZzU zrsgXT5s@$sI8?2ex2JB8n;b>oVFzuc&Wi6EhZ!_c^@Fzb$aE!CO!^`zH#PVVc1*fz z$<(+}$f&d2oDb$3Q1XWISt1%(HqnK@;aeiRxsoB{`rPOZ#SwFV+<>heNz6b!h5=F+ zs3_gq8jo=dtJ^#rP6)j{thZ)AfGOxaB(~V24-KG*qoIemv4o6dm&aYT0xKJ7iM;*# z*o(NM#u}fhq;E9~Mb40-lUp}_qafcCY%KBxom#JrDjLBaZ4sw>xv@_p#AcFd%NHdX z3H=)O@PSv9Xk9_^@0l722-!~vyi4%QFur*WY#AHoD*x6TBmx3%F%uFZ6h@!qgSlA|InGM#&&DGfQ^Urlryw%UkLT*9w`=KD@*M6DT^&5hn5f6~wdN^7 z2Og2mQyc?quhEAhTFWwTy}!JxZg zjct)1)kxW%B*HFZa&y=+xpr|E#oJc6B$E&EZ#^bpPh7+Um*$afBR)=_i@+f; z^1?+UufVy$XJv7y2YrKbPD8hp8vFMBIafB+I(Q%WGQB2Z(AO0u*5LO^mwoAq4sid3 zJWPJ4*a0xLd!1Isb9+ftAzgF3Wul&f_2W(eVBz+Ly-8bAHgsBGNbsFq$zSPVO=UE; zRUyV+v!AWzu|bj*&P6vazcRWlZqry~G z9rK7F0F|#l;1-$?0e=5**hn0D##38OdFH37y&-AS&0cbR4lAWxWv2!uA4Ajcwo$^+ z9tmJkpki#g4lT!6)}7pDWsMA63{HIn5VtzVn_&^WXOf{PITi86zy^87&!loBbDb5p zt`_a|+=JSgaW{1)Kd-RA6OowFJiB*A3>{K@1RU|ex@WX$vAlP&&)Yl}?gMqY{Je&K zlu6-*V3zzpN7FAhBC5{=a=y@*wqrBBk$jlNkG8Ve%{lqVa;tHn0YcdLR?1Xp(U?^t zX|($BIIRc?*~^_V0+=3~=bB1!GZ!co4K({(yzJ9_2a@W1Yakol7sEkULL-PDj!R|o zhSYoaz`Qn*#nJxAn)LkVEfNg5Q+nS_R`7rt>>wSIN<48^)eHJT1LZ!%w9JYKI?EuS z1N!9%-<92MtyxGTJxlI|dUba(pU=W-YW^?I4$4X(9K*PYuju5Dapp9;6Y%wg<~^9a z=QE6*wP;9&TzUuKfmgkY$q;1$8LaX7&lxTWQ~+oJ@VWtp6^U%2$5K*Ciu9Lqct{)fzAnIg-gJ9P-iQiVy0+Bf}4Tsfr ztkkr3r-EZtVzufnqc0TX%9~hB+rxpf%s(^*E`^v!sY5j%4k{GI;`C$_3ll#4#{K&) zy`fiL)UUga(`sp+u=vdPaE9URuie%aqpgn3VyX=sYU(YW*zVoO$=Q#yCc#%|^LZY} z&NoN~*C0imFMO$5W6GBm>U5LzW+Uef1V9VH*1}J?FSah@(Z~-mqvnwh)6tkqLm+cL z>t$3Q4e~X?#aqVX9%=!92IQzj37uhp&v$N?xrZqF*JK6S!N+h&2p*RLNhSsXCWmQwfL7C%94@Qi|I-5KGK}tzAO?d&V=< zKs|YJOoKw&&AomqXi(|zd2f#FOxe*ZhoZT1>`q2s7)7Dsn*%tHr;0fWP|;qYF#gb- z7%3!SZMxi%9qEsb_Vz5A{&bdx+A{Wt3imd`Y+GLWcRn5X87kF}9^6h9v>sNoqdOn{Ye6r3m?2vESVNmSNUT#wOJN2N^8|*6D0Jo{7s8HmP#V{mi2;8=-KH*XhWsw%f4art=j@dv)x{-tM<w*iI0fj z#(56EF{#J&5CqHig%i=YIODTwYqcExMxCdE9kki{)0SarVi+pH586}R`YMK?002zh z_;yV}9-;+8>bFf(k?YGhw(8V+(vtET#?$_$M(9mpQ3Mj;NkM&^W>cXC1#tzq3VLbf zu*w0j!y!w9ttfD0YL)Hr@wZBb@3OJ+SW%wveW(OX(h1pgoV88Y5(#P8%O9^&Fpdj8 zbU5x*tCl*2VY}|?y|bXY^k->k-Owyq#%0WbkSE6T1+3NEV5JvhQK$o2NU%jhd!RY8 z`kS|lNAi`DKJp-D5+1od!~meaSt6f|Y`CMa!9n+bVEIj^qGvu@K$xwQp0K??!*xxx zoYMFo8ppCT$;|e@e4$cH{qMnUvRm^7GD!ZsF25aukH?f=5OcEVsmkdb*zGOHdf@%Pv~h9d zPt-T#*pSf9>}e4o=gz)omYRK^It&d?2E$8U9?@iLyZXZ2-AqVRNO$hp^j-UT`*nhg z)OlMg%)@GoCFi%q{)bu&mQkH(5pFGIzq=f=&~-2H9MS+!2c=;B6iGhR@`J8YLd=x} zpfHm-!WkX#`w=uc!Kd-id=sPFXr5qu{#EO+rTe#zd|MQvS41Jjc0lzUN$ zQ?cEgWJjHjY>h=~g=Yi+(e5)OKMaXc_)Qa!x6>EhzbIhJ(2zDXBb5WphLIfSgOB7P zy8svD6M3yiEzg`hN7f0rPKE_Pv{JCMQXiv+Jy0ERk|j|#88PVNx8Ca={L(zwe5J1t z%^geCiAzf9ZrG+alpcrD_fgEcRz&CA9*b4LP^jtO#X-00lPoK$$iB)q<#ff=Vvi94 zkd4e?3#mb>B#Nb-ZvpKuG9&&6H~eZ&*0Eaza~IFbY2_wFi5PzlO;? zPa(qh2g^2WDSkP5Ns|UO}B*ByG#LdZHy2tC?Lcc<;;Zb3ku9 z^U`Ola-zX4c9l4B5E1jkqJAZ`+0l72)n}Nxb3A}I!R;UmOWB)=9Vw5+;aMVv{Cuk} zPot)yK9j3WP$!U#^^eYwqglr z8r_P&AjyiyesmBO3=ApNfvCN!4M&NJEA;i~8R}|ZPGzQlJ|G-HND)kMiAF>->f*M* zmEs`DBmhf5w7*9W+i3*sER~XzuC<&g&mHlq3Gf3qCSqYje}ZA;ZLQdMg~3%C*f881 zy~I#vX^rVUq{ql|E(BlkIcX_MB5lTB6&USi)DfosMsYUa0*1TJ7)23<$}+CgbH0%s zCHW6fF0Gv+%SnvBJ-|iPD*#JCw7==A2$GtruJd@aS?Eb-@GdjXfRrt-pM72`WPd_4YC9aZ!r-@dN zjg<7o2@;Izs_oY;*@Fiej$Y?hD`hNYBes-QrT4B<+rr~={o!UC1;c|ictzhCz0&B`i@dr6J7@ zDC);*c*j=FTu?SJ1DB4j8*tVLlcpWGPLY}uLy4ijknw_UcRISyEfdi>k=2fsa zd!jD~^DQgzWp4^$afZuvS~ScBurs0_{BJ=vP%cg$0g((Fy`6u6v_98EWlGf|jnZ@%^N_=|weavazq z8Sg*r+f|}F%&XFaYqtJ^J3FB*bRTrO0r!S+_PJmc-b@#!y zUyByAP=GD1vZdJY&la&&V<+DH+W^ag7iY!oImHTAmaY(ff)wdF^|^SGaaP{owd#@P z*^HCVnn%^i5!9hna&7%YChK%~@DvcVQ{{HX7YsQO_OF~$(ua!tJd6Wf{1>bC!2)!^ zor^HPoSd`fXL_+is51-rA865BFqj#UgYp<@04ev}wjKfzdD|PxBA@1A2B)}P0$a}* zn1kBn!8@GeDs61=Th^a1vI>ujC|$`TPY)cg^&t&m@Qr(u3^vW zC4Gc9V6{4Sr~+iZLpuQ)fahIe7N$?!K=6Am3|TdJ=+xLxJ{;svfi!_lnfLJ@e1KO3 zo%gAW#CqO3gagh2i$GKouBV$U>9`9Dbo!xC1NYc1S{iz&`k}o{a#oqI1QWr3n0-{e zns{j5ONYUnas`_3A|KSB58N{V|k)#^PRkuCvslc=xIfiaOP8I)5 zD`wD249g;%1_>q^W3l~!Y8g6QbxOFzO{_QdkuzM~ChCp4i#3oN=t3BuFfI|0-;E-g z@{BNYB36895^5-kO}4slnB$Q9$LaV-ERaV6|JJstxbzooC_$v=GNe}q(aOozKb0@e zmDQy_YAY$KULm&iXMHq$(~|3+hIBh#GI};Jv6m-A^gL&DJ8VQ=bFf9ZnHgT=#V!RAzRB!@PK6;Fj6q%;u%&<d|n)lG47T-U#_n`m6((s8&~}l3-R3nRVePwGM>tCDvcVRXDr>B+xHVD^*n)W~do6T_ zd^A4KiTkkU@6J_Ay(W1kP}jFR6v7g!$5p_-0@j{~%@yhb7gcNXEAQv(IlM_( zB;r}+Gw?lCi!xF=RP-V@s5QV9K-D`6uhM}ryBW#%mpSbDqf`)HRnJvW`2~WnU5(}? zfqn^*s67j;XE9nIxfOtNL3$8ZDLcAaozGo3oo_$3waveGDo5LM;9?I;Lb6@3^c@BB zOV<|7&;f+ac@9p)+I0q-$t35D_AO-0cn8It}`f0R_ln%R}89#w5pj>aF`1?gKz zqvrqL{nVHH&UeutHzy+%59ZE<#R$r@4s9f;ofMEry^CqB6zZ<9q(E?k;B{}cn@4Va zNO$Ce4G#>}xi3X6P`zWBN_X?r#YUmocC}=Xk5BiMAQ%;R#g2H(yOowiUxqR25^eVht$4Z8$|FP{vm6XT z3~YA~f+157f(T_JT|iR(v{c$lgY?SO`n=PDMX80HS%aZN^Vnm?2n#N$ z&{kL*nUt?79mJNx{1jL(TdL`e^Cm_e;9sx=)8C%Dwl(#3cV`TR=6sXEib6FdWJ~|J z+A~{-<6iq#08OJak(6~_v;A@j;<-XQ2kTcSH?B6=kNBFh*$*k|oU0p?jcJ<^p7HGl z)y!Nxtt2PX1o~{$7WDkjE%k6@5?vr9+5OF13tMOhdM1K zaRCl4hLXEh33XYjtm2lwn{#B&D#VoUVK~J8kr<}|kK3SpBZ9z?oNJ}n=#2C|3C2VE zdYt3pP6n?uKCA3}y#fgkFcmA1LQ;9Jb1uaLh$8p*$arUDwK=$9Gp1!rjs5Y6{Eo$X z2h5_08My?!-bFjZ!}!Ahbjfo(S$+n<2Z!<>Nzj33ybuL^co~q|vKxM>+wG@y6Dq`` z$erojljDwJP~ay47l6|#hNREWVh2#bpxXI3EG5epf)ye(X`WK7BmAiS6N_+=KbCSj9t?^7;HL#g9h*su7 zvvbgy0)w&I`>wYji|>(m<9-$!|H|vxWl?{CKMJ`!YKMiQ7j&*#yZ3mF26gAv1qIu8 z@0ER*?A?$_Bgyl|4p>M57oe0oOiszaL!&JxCh@;)SXXhX|MX<_JNy^Cwatg2&%5rs zI0zuIR~+uTm;dFC=Fp-()M}^9yKy8H&l`CYF`|Io!-G|-g46l#Q?e=@7qvXy5V1s| zm=v~HF50^O3DN>pt#eLtPUd{37jgg3+aOH#G@w7wPSTsG14%x;CIzC>e zzfyrFV2&R~Z%g8QL7?|$5bXYRykcvvKV)b2i-gHG#8HA(@Ysy;LHlL?0uwDkmi0hB zp!Nx|;aM1&?Wc#?*lh!|1B@|^5?{on`3nTr5?rk&whDxO(OW_)?0mfO4YP5CBiQ3I z$Je1y=J#v3@MymWlK$tF-Or8}25lbWi&itQJFPDD$F`7Diy@jAgJ4oG8< ze;m(EOA%D2V}@V2Ptdw?F9j+VfNznn>Zw7ZzM#eF(9FFcKYI4-yQTj=i;wQf8zxL&yN`ZOs0F|mPcn!N+uffUx7ZYIQ>>TQGb`!+V=HD6PT5$W4*$Vm z-1QyyZCw6YpcB(>i0_!RP?%amOV>6+lt4h;!X5IEhVlHA38S_@0pQA#CNVe+*Gr$} zX*MbIHU(6t%N_9QG0?#}c#GY+$?xj&%Yvs=$rTfVCXH5^Yz}6V4oYgE-U0c&nWPFB z64GU%HKSG|WHdoTCI$Y>ZnPrG9Jk6T}cdJ zEmeRdl)6*%Tbv&k>LwSNjL6%*V>(93%k zlO~~`w*Y5wi(d=|aOTMRfTCZaQ@Y$Yp%eH}m*@I)-0a2pTJpDDjH>cNBnvYiQre(* zC~kJqhE_2V>ZAe0pq+|q`}wIvulfH_uC7O%y^E7RYT zA*w(xAoie;jgLc5n4vXe%W<=7I1#?48305G8lzbs`7LUK%K&)b@iI>c0zYb3HHKip z)lA6)#%1Bv(iy&>X!7VyHg-mwx^87gZgG}ELKtRk33Wr1}teaH>EE~sJ_q6 zlhygyK-U9d6Bv-gujfht&A4WKPD~5VoWMpXu_%RU)&mt7(G((Yb)g`_Md9KXfEVEb znZE_QX>h)s_~}v>20-{}cUxdC;v|vTwOaqZ!W5NXHfBkq2P+nu++Xk3f`k{(q_{G>)Vm=Vb^I#$j8uLbelUOP(_xz0_h# zimh(WhiL8G3Gak3ILD=N!z9hOxr5O#I=CHaxh~rAZR00j`WWAv{!c^FY=U1LD<`Ot+(xlHM{}dtK3F|iAY}z*j-6qg`D-sEr_z! zXusJ*gAcFRv4z|JVdB% zah%MePx`Mn0q&SNxlqg7{4ObOA7PcpjM2hi4b+74j9KM8cR&L{&|fAAuM5Sobwoji zqH62&$d7g)Fh1)1CKsI*(nB)|cY;=r?yS;9Ek>;XO4dzm5Az-1X)W zuOBIUc|WiUAQsk{b|#ljKylgYwR7CWYfPlIZGpy$5Qpx#nLOLccSPo2R9S2a6QoSI zn^?uS{M3k?Nn=_F9P{@Wm1XU|CT@NbQhY4=Z@+Uo;IyC(+wb(Ixb^6T>UE@h=uZ_P z0cTOe8o|K^g!lpxt7FH@>a+`IJKtzlwLH9;q-`{z)MCFlW#?^Uh}Da^qVXwviEiB8 zj%f3Z4#;NZ(L|K-1{pq|)H4v7m%Hqz**LM>#ezGmarr$HM>4&HkaF{rYHxSZbm5$&EgY% z3BmASWW;XuTnk=Lg9MW=4BiW#UIs$}CRCoz94XHTV5rdgZKK>Q9hi-H8XXfbtPSFt zB@Amw^s5d~3aDMG2ZJ6jXZ@A1v)nbDs-hN}gB~WSV*11JLRAB7oza;O^8EFNbj;&( z!-!Sv7u&$J8;YEPJN}Jn8>AG!Q1P$Cy-ZWU|$_HK%p)7QXTqDfDuN#mUt* z0A1)@793_R|3f~pw;}JzqErdrvAP}4$SAbADf z5Sz#m@ME!J6&?j?pEJVKvyNNIB*IjEMDUI1m7~ z`Sw(hNexmUqO;!#*C)f(5v0@7G%H_Wb3mCt&~H)~ zY8BJXO#@IzZ*9{VGvZRVsLEz7y-2$+nrIg-oU#-u*{Ktu6*%$&zQq2BcarBWEe8rf zp56%w7Ygla3iB^!M2=0#t*R3&^xk1)%Yj9JJM+%^JmE8@+(a0RIV{$KRw{0!CyfwW zEBPb76a!z34QuXF(P3CLNYaGVY6h1 z?#5f_hS)`~oCfTPOve}j%ul^Uqc;Sw{5sSyMd%5Tabt~r)?5#Ctw#BG7v3Vb74rBC zW3S$<+%nSyy0ZF9P(%DGx^#sf_!wTbCT9T3`!o*SsbD3+}w0N(V<$+@gwXEpjp z+7qNT1n}EN`LX05@V5{9Q*bI6)ol)9M%HlG>Hb8W+h%4~N?oAln~Tg*2?wJH2g+lVf}}+QRETH$v+ep^-A~*ScE) zC6LfB&8H-VJcaoDh}}rZB_&bJwLZ~W{3tQg?n4unpa^{5TcsvsQu?~S(v2!iCCeWF zhtO7{+KnO2t!k(nd|jfZqe!fW8^DwHEvp0 z_y+LvgJ)Vlf1Mg^d&mLS#=>I|^82ph#dDY9HMDqKl{l-5K{^>tyY%>Y6f+G=WAFyI z&tEQE)jbftsE6byC0D>H3R?kTBWJl{k+xXDZf(!nitU5%>V#_e0ZdH5Q^q_eG&YvRf@) zX7^87#^aH|-t70$p2o4j7-GPVc4(=S%y|UVgDn4!z2gJ|kPnU^<(B0I>-atqinZOw z{hQ=fLp99ou$phI3WusV{X~8#gk2)Qrc4&c^UVK9KjqiFI_Q@Nbxe!F_V0O|P;DvVNc$o0r;$P=rxRF8Q z#N4;3m5=wCDG3~D&k4g4SRN>R9Ohk()ujA8K?Rl4qeC~7(3^gA9%e<8Qrf*iRI`)B z^N&tzRrN>DFpejueXRPH_G=wvPH0ziah{*WFI7L-(vMIlHXZ`-yt5bOn4J`Q~wO$%U}&g zD8gx)^2M4c-N>Q0NaC&rHd`{xTeDX+N_Vp{^dOQ^Yi8UjNo z9qRDi;cNo3Vp?+iNa2NNBQSJo8{SV`hbxq1;}#Z1lv1sk4RbAh!Bh9B3Ic}I3D5=u8>D(T+D~UK?xog zHtEZPn2>&pe=pJ^Y3(l9vdPR#zpF|HrDLnHDbK4^`SSgr{N~m)2Wp_?oV$YM24`LS(#3ynIn)e}R2QwUxev+}^ zSHyS*Ka5@VLfD;~eVnGcW)k%0aBb{X{(F-Lq;8ZUBDhFn9S`$Vgz5WfPq6N_ z06Xr(K}0!gAzdIHaO2Yy^`@M6hHHEg157=^W`4(#S=W?_{s4Bd6=5Is~R zGc)K*%ESCMOJ)A0;zXh<66R4Pt{!wrp+cL#Xot7?NLp%_e{K{r?}0wZa;big+ajet zF`@b(zsjEgsY2t`(ls;l*g6IJlz#)uRoXu4;ra=Y@Ch&TmmV7Pf|P(AEEk7TMwooV z+-4z3^-x;A*U+}zNF@W|xm4Zw)MZu%pJY8R0*PNj({O$WH@7Xu9xPwhwMV13G**RI9G9356cKw~xNwN(HPa)bOdH={{9&_KfmJJ&a5o7Z7 z*h{7&a@M^={?0MprhXsFlTWgA!ge*Mp900z#EEom`;G)mA}=OE`v1S zK^3!8NCv#2Kn}mzJgH~{Ks6ML{dIvo(}${6^<&4p-%b?xI6yt}4;BI@4whR-rDb%0 zw~BE)CpM_UBr|@56Ny;yP)i4&tTZwHo0t!>j`94ZlS7@Z0G~nzwSu%i_SuEyVmD;S ztUA(`khZUG)<$Wv#h`b+5q;Oya;8GFTJ+QMZ7PmZ&6Sly$u`ole(0lQ6=T`*-YRtf z9LL-En{d*1*c4r^(^+pe{Ch4*(E~=w`ZM<-WqX^gMc=M?r!g(xUZa3>K#o`b1u&O& zKUh7xW&!*v1xN)UceK{y!JcSEN73Ny6v)&Sy_`6)+H!~AAC6`w*76{|WuC&h?djt| zP$wM=IYh*!cF$cbsiGFVJ)Xw1d<~v*4^cs@gF}o7xk^xt z0dk}Fni2g2^t9I&TAwK+EcxFV8DO{pXVLMvC`rl)R>y+w38F8Y0>H7gBySnRdY10g zJ7O_{o~85BTcGg=I5I1U?&V436KyF~0EWB;A({k!99Cy6<^*~;0bXh&m%aT#E6xk@ z5|ap;|9(9l!LykVZHZ@z44$@9Yeq`?fG{&|Kc>>v$9e-X5Yd9LIDJ;+L4bz=nIC@F zPopQGoIsb!ZnR3+4q}jRAR}Cn!tH!xOCMMy!k>=c8M?OzkBnib_|=cK!XY_vk%zx- zm5HQ9^|?4wvu^{g3UQU$doPZDVqEsI@&^XNO=eu9c(hTl5{oj@iaNps843KEK$4dw zYmp9z32@i2^tz?$jTyuL;lvD^1Xv2R0z9p_Ytjx7oBaNG-5|IxuAH=jgs4G?KnwbE z#)Phpp5tW4B|3W-@4-9zi91b}z>6%&j+9;IFduW-cnZ})66*-B^YGehE`t;%?*qj( zU2Yac;;7-vT0I#Gh<@LMf&n)f7)Ac!?AlC|*183Kw0AH!6}qYlNJ z*!TYUo?gIvZaHe(=d%|ruh{q)_;UX>Irw?4n#jAQ_uQPl_dQ{&Phb#8BLMZ>9G(ND z*-sClna_6!iKA!6W5^1MiGuBGG~Ton()|-#it4N|00XnlkmgiRVw#SZi-3?|Cr*}bA*ELFN zLBMrkmC`r3jMqw4%Wf)xBh70p8gA!1sfJWZf!H+CM z%n|ko%dw^soi4X4hMxs9v@WD#y3cf1%BYe8z}>Qp`@tFP1Y9N1fMwC&=oTdQ^yD`& zNi0`HN}{s)IxR(?#lGZHjBBz&u3bMgRylXA1LioC71Z^S$>octA|rlsqn-oreyGi2 zU=K-s7g0J-i($A7X10U`kg({r5d3aKS!FQ<;^jD*2N(7b>6`Tc!Er$K55)Ql4!oxh z=B(`})NwNX=4b_1R8LkK6V z5^4!aVAWWt5KkC#AgO8Fy!8b?zi_=MHy;y$O1qB@J)|Vg=hN-_X=OT#^^s@LdGUSO z(BQ*F`O5I$|H+HOwFSvnM(xUcwEPh6wN|R`MnX6z+^s5MJgL$4Fi{bsvZwOH`y+n2 zFe#G04k$y??8hhDteu8ii!V5bT{RkNHvxhgh^A8=l^MsI|Bp0lsZ4sF8YuIFn>Upo zFG+-(kZgr?J@__ba5+-xW=wacChoQhX@$n4zYIS+y?S`Vr;ylh)K+asODO>%FKFn_QYej5;C_bwd*Ut#<9S(odcPdVJX6^O zCWW!6L<62kbFhrkav=O{2tgZT=J|$SJjRvmw?TJt6r`XlT}?Ik5~CxUN-x@!&KP}j z4$NF>US2GDj*zbM)~Eug6vFbl`eUU6!sO*s9fl7*11RZc3rwRV;%2QnYDj)gfkzp; zHe$ckgAK?#CfjFz9m#FXG31)_1?=q1{oVrqL;cW$#)DaLOwGv^#5?55g5x%2tB%^6 zWH)kB)KXBfA1hJ=-!m)>=*8Ns!e|Oa;TVGt2Q^;n8OK8UD+ChdgHAFjk7X4Gl^MVc zYL>UPVV7C*!9{g*LJX_Sir1altf)3dz^IX9Pg(nowRX;PtP_peDZq@LTpoP9uBrxs zqX(car7`&sc(8EUTR~J86uX*Hv-Zb(MbB_C7NpvWP-I>>d+IppTFia-MDBJehN8v}Y)vGxtgmS2A(V%r*e(PEs&VLILuolJwDoDGtM{{7kNBbFP>h&PhdbR-D3P zvQ7z-A*GQh*TF?Mj}y*0k1gv5@~w~P_W5n6)$fe}3RoFi&Rv9=0!*BLwQnCh*=p9g z?fZQOw1DnKly!Z8ijPC?BhWK9O|z+@cqNR`#xuafXvY3dfwW(z*#Sjvh(F`MU8eCe zemCqk=#*sWtL&Qm&pR@a%!Q(ha`% zlPqAww2-4P5REpbklL(YLkIbCkspazWM~%rgWi;)BqU^4QsFTaj$)z9u)wBx$x#*! z{OS#`a_MSSD<6|64s2p~i7!!93yUd_5iWA`Nb{1RqzQZ8Pu@Pfli*(nzDBS*wcAK1-| zV@Vq(wgva*8PPnBlPqf!k{Z|x=;1|VW;R-|kadM3X)|3m-Nrm_K!Sxg`-C~F`;mp; zgpMAJpC5h?hJQbo8=p33lv^sdoVhiPWnFQ*QFiwT%lqu&3mr3XK{?CmhC)* zPi(>(iP-#6wn31=CcBe3{gb~hr<33RV9iZ5VP|DmU$=XJl||LAv4@hW#i*-)U8}6V z2Oze3Zh-=fD?`SGG}b(qugf?gg|H~6`P*TaNz*Vkk6&q{!{zIH=!W343RY?CA=gK0A?o`{Z=mfuPn2PLrGyLoQ6t*!rUGmI0}O5rQJPh`MN(3R)vR zJ5X3iDr|}1^zabKAEbQJo^mjdRULi*#Q*G)>%0neftk4v3+hPL ztQolHuBj}bW|nSmGbuUG!aJBPjbADWyo8*|m#0QdWV2=@xw-2;bNESMK7`JQx=bEw zV9r!+TzGSR!k^;M0OoJk;rq4ctp1Tn8Ha^*baeK05$8ADo4eMSok<9Z;mawybz@uh0JrE<@>`=O+>Prxs%sF}VrOQ?EvK4RqxSzBzdqGY2cjdK{f_=wCs3Q%-joS30WY@Z@B-7 zWTXvU)0w*UJ7%U=IR#QL@A2+NYS=M5cmwc14FCe|#dJW1BGW(9-2%%bz=o=(cGt<@ zxmt*9D1BVBVUP}0{VR@B6c+DTYYvSI7IiOJ= z$|cw1HIoeAzMdw18aOsZhkaW+-5;+>=5)|qv-@$un?CGw3;EC~q1MK_!5bg>;JXar z@4)E5O@|e4?o~Te){~V}Jfs7o1BGybHw!m{E4o)?KPxse)W4`2xrhu+d1M+7)uz*W zv}O`v{&yM`FbpLsgk23P-5;%aqk@M9&m(EVSKi7FL$fUy@*@>seibUZSfT=JNT|^a zaUqRs6Au~8K`me+1GR7*z6%)Ehj4PEgGUFiJGd6EooH9>-aiO<3iu`^I_lFPWNYP-q@g9rdy2L5F1bt0vn>pkW_QP*In-8 zf^^-ZKV<2W){`}p3jI7X#Js8JL%KnR@-P|X-7utc;C{=9%2Op7uv6xRAwg_FWH64` zlMOjAG8jeF^{O~sL|Uqz7&XH9u?yFpwH7EHJe`%p-bT3 zAIllmQ#F$a?Q1`c2v`USBG}kBwA1~mnl~b-Z#9%Nuzg%ODVGu1b+rsCHbA-#9@y{` z0%MggWAg*7oRPWWf-Vz5eJeD^K?WWX^Zo!b#ElHnyF@=D%=IqOe$$4-KWN>6VN>AJ z9-6UA>!F$%6zpp|iw*Nq+nf#ij&`~~RFgD+sr@w7U8NhdHCTNc>9D;S!|6~^mu?fO zB08{e6oagwz=wXleV`zjOog%q@&P^V5;OUbz=xil53Yv;J-~-e4;RyU(cvLJBt{41 z)(`Pvojc_K?s{K zAns#V6BQES7VfD6At;C|oVyE+r$wvT384THyxy=0;=*OUp}vm^x7lcq{aV4ET+(R=}Ab4ozm z7`oQifD(hLCq~*mM$~wnM6hdKctHgvXe3)WfrtG_s8x>Qlu(5__@U9ge-+YilGOj< zv8&!miENLRsMn)5lMwl9K8q41xT@Z|-Jrz%q;fbVc)M^rzXpTI$q1|<10%W!;iCG% z4p!HLA6vlQG7_R?Cq%j4M^LzrfNC8PqLUD4VPP+K;xjHn6t=Lq6bXbFiV^wdB;IfC zS>$&|#YTz#juJze<$SLnx^S*XZYCwtiIga=tDixAf8o48bmIaTb`6LtDS@8Uqy+Q_ zpoD?|Z~x@PP>|>>MK%nt>f9JKr{!^WC6+AQN-$rjYr(M_R2y}bKmj{@F-n!{KD_=g zhEw7sd*M}-*dH61L*>fY$1;iaAdhSxE=jNs$i>5mZbD%iu$<&WGwDf^>*s zix7X-1}d-oBdG_3e`I^YVqimdfFJPU1+1>fhG38NxYuJhGdP5=@hmpf>?Op+K1_|(is(0`~bilHaFnA>(9Wvg%7&q5I_^@83l{M&|k-d#Y zfn3QtGC+squ7Onox!#-e!NbF!-+PxIbDUo6C4eCsbiVbm1Y$jP^EJNy{2ET9gU@GL zyV2slKXmiP2Gt={hUlz9Wu|i=LkbGjS!7@v3!9Fk$iSiQsF`nYA*8E(vPNAol6SHA zG!_o!VIy9mBcI4%MP$%f0e4@KA)n?h+qijWiw?zp1&6m^L*_{`^jaSy!#bM`uRmXZ zo<#;reK;F#bhz)&+`O?tXNB8$b>@fI$(%`t?Abm3cwns=Ds|Yd?q>x(XpZosY%^M+1gvf3YO+H}v9Qoi|tZ{UJ50!MN+?Ctl ztVlX^ZrFrKONR9CGY1Y^6pU7g4(M`o6JlV=I^>NERcxrvhU#>vpDCWj;%7z&dn`Dt z=WZqwzQ2AyiwpBM?_99`NpXMdCi~cp%!X@H9@%Nwm}uApE^L-7L1#teUU69SB$0uq zI_?`zhL*1LhN7!x&=A(3IIdKZ0pt8dQhKnViwqJxxR5%x_*>f-AHHT{;je9?&WA$7 zGLM{O`1bnkG&-0o>uq0n$l>ucQrFDn=J6&vR48I59V)LGkVujai1!g`Bh7|epee`v z>uwqdY2}qTuCk<&nTCN5tX6aN9Y%gpwrs=Mpx9u3Qe=+}Hh!B0&>vb+phd6Hbe4z0 zLtYQwOg?-)j}JYJjSu@icfLP(v&Dz1dxf7AC>_V7Y3^)VEfRG$fDbfF^K0iM0XL)K z@8{}-sQG$TYNd4q1=wN)a|dJL0V?GgQ-)JwL{UQZYV17`CCZ=OS@CBqMNN%ZlPksj z0Fl?DH#0oEzJ5K85Y?Afn1!?_#QkK_>ki)H%+3gm+Tk58Bt%mch-z67f+i?>=12(2 z+YnGZmn_%r^p&d$G8-MBLt=z5o)?+tMf3jVFtNeUi(<}(lMW1CEn?x-!g!yUv>Mca zz5g|@+7j|T)bP&h(VH0^#_MSUxe;7jKOdYu9qy0bywO2-3-^2MxcEh!43&K7G^@wP zQu1NO9r7V_o6+P$uF>-i@ge6cF-+yQf((}_@qz6uqW~iF#4uH=O&YoGgL%<4a=HF# zH=sY~7-!6Dq2UT_H}8KKK&_4C4oa(P>A zW53yV_))2xF<3#-mmgYZlGo!mlMns%G=g>AEY)oQVrryRQZ_xWb)yc%QIoQ3)md z@JI(~A7a~rLppm^Wk*tuhx%I2qC<4|VcVO)y!MIyK(Si06;bdOV;LO;NUh`>eu;^m@c|G5=p*$QNbk+3YB&}ajjfMsy zLt@UWCVUG?2-Iuqe6zCnkfTO5B6p)<*|Rnmyt`{!(ZwS8abav^aWx){e?W-fxnxnJ z+b8s*JJ#o9f05c=M?}1QK}vK|BFBeRa(O+CGs8oAO=nReLuDpPm}ZltHw{Y7hr}K?#m1Rb`jpM9DG(;0|7JqFiq7nrFgL$BY z8W=vN=e6pTf0JWn(?U@!|eJ zPIkS$>8i;tL&tk09i%BnbdGpqIG=>DC_-q1MN6u7W8^nDA#%Mk|1c;pllCWp5cGSf z+>pmN%P*KDgfl{f2wRBwvqHH)cVuAEJOvsL8C*AMJBj!OX>HI4}AyE;Lf1c`K_IQekGLpaFIL zpn>`pKBt$wOnq}R-fvRaw+9MP@NI-NBlO33u+Agn_&I)_#e)J97x7?UbGSc*^F{+* zGx_LtYQiX(XW{_q7@x`FYDvQ^H{O-u_lh=jN??$JOo-Pz42(`>34?lP#~W?#2c~a)Mz4A1;Av#lh*d5M5Sil?;tuG* z9#+#h4(Jf8=uo(MdC~#)!j}PpYm^d89pXbH9}4-vGo!dgwRLs6gOm>`ZxJG%U8ugp zhs^HlXb&A2>*<>r9KMckXAxpsdt>oNm-~U?n~e{;XExD6Q`+2!o83Y(swDw7&{hxm zrpS<5ar0u24b85~k}c{^;Z%scQp|`GxJHT13~y+5h6YamkO~sPq|rWR4D{%tJWn7J zeX9aSDdrbr%240`Qa%M}w2jBIhxPEyj18~j>uF@LO<+IYpdA|?7`|~fI2p3o!1>HD zpO;EHR0-6$cXiT1eLcjCDn8^PKCorjo#{ls-iBm|R(%zU!*a*?_dLK?Z| z!+FUya_ML|Zyg)#H`bvf_RX7-+#N!!$8Si8*Ru%GqA#|vaHGNf!2Hcd2wgCmCl8P? z4Au`p2;)S>m!g_Oui`_Ceptv90)m_TEwFqcpbxK}3=Y>HB9v3l5L5m&jq=OU> zsb;4`{j;~tKUv zj&&nSk_&@9tl?V_CK;NqOUl8?Fi*}aG888RdRVy0vOZlh$bcJtrYuop7$SqGk)AsA zc*91nytjiXmv%7s+&?}nu&;S*wLQdzbsjlJJCO^ed&45!=x~pR6UnZN1q#q$13U(Y z6m63ZLp{yhboM@5Kmdao5A9Dy zA7i6~GKc0x2lLFKmEYk zyrGK)9jYfKDqUq0?I!2{O4^>eUKJ;OB_%l63rc8BQsZ8dNr}ka$yb!%K@{2T0wpSY zSo6ySz8~mrxxpYVA(cu@r-Vc>$=YvHMBOV>FOCa~B!snx#<9wJ{$>VIg_LsyI8>4T9E82x+Ur~|JS8Kng90nbQ+Iwgq@nIWXj-*7(k>tzPq znUi!XfetJ@7>|p|4D>VHT@WVNWP_gzKU_kMImrOaSdf8}-cdWUz=kX~^x^|Hh8t7fgg$+6u2h+{dr^j=JVEOl59-ORu+(LId~=DFm4 z2U58|aLMS|r^S6z9&6IOJs;C9u_QzW9?%&F}L<48pYcN7%0F1ZkO&x9VD zTwvLuvFuP78j3p#Ho2C!D#Zihy~vQ(Q+a=Y3?H7!vq7;I*2h0^%Pi}un;96wd1Od1 z)Dj!^-R+Vd&m(!Y(8U5;KrFZ#x@0y#C!(_f3n`>r5F0oX5jCn$Hk9jK20e)lsVFi? z6W0_P64*f9JlQ}sRP%|v96U7KoXGo2j|%kQP&$1qHms*^M(jUFIE@a~S6fVU_eF>M zb2m;09uyWNhe{Bsa;BDa=tY+b^rt5sQYIS)kA>#;E$D8_NpP~^s(ID3ZWPhf&FoAX zc%jHVCXyQoHdAEqdAuAwGTdC3_eZMJ_DA*^puSEfgxVgX!a9Qt{YU?K78P>v#TIqO z);$&OXOJ@du#iOtWsnIEYNtF9WEh%;OG8D61fLEZp)`YP6#$PwaKE18WP=+CMw*LV z#s(Fj&?E&L=m1kYOAeU_MQj@#JTye7!xkF;Qb(X0bU=L?4?ua04$B;}|LouUkJIQ- ztT`R_%D8fW;^vJGMRf3^;@zs$5;%DaI&>8r(hwa?t!?-KDmawtbV%0=4LV1=0!x?V z17!e>Kytq?UEzU8MHtS9M%hctsyAd8#--eTvw7Ty8y5J^r~rScw_Vm#H)O>3(-<*) zM6=FRe$R;enPj6@Ei^#Lo=LAx7jaNc>s7L%buEJU%meBpq-B2VZfT;5Ixm%LEaFD6NYR#;PQ& z%ai!fueTX=?sX1$yu}CfxaVbteBfKtq=rIJCLKID6mMds{J|FYZ{3`S1COQjc^P|* z4(p8aYae}_Mu+Co*;;QjxbF|&Y;<5tasH2Il!J7ropBA;Z15o!@`0L%$Y+(S8%YSYuzZ6a`XnSo{9W?6{>PNm*uKCb$zxsa z_3X`z4*fhrSoF;nwchA*-ygmy)d`^+r>YMy2!w}sB^^xrZs|o0I>6KkW+*Zvu3COL z=}@!N!QypCa|F?$sD%|1n%p2E=)m{U+?=|*JT9t-l{a}@EWcGA_wnOmkPaz7R*PB> z-b^~oBk?pkpwlKg&9S0B9i5aW$1_0evu-`w0DkyMEJvF7%A z>Si7k*4tTBXj_rG-so-LAG#^#OwhF=DRIb|Oi0KS3+@7}5!3AfRLG)2QF9x+lNA~? z%RR|ZuJ;(QfIZ)hEFE>EI1a%8WN3J6ia8e)7LjGO*bRG9=U`C)8MP zMKZvN;gSqVWJu~@!4(7}gghn!-^Q$@p=+^b3!X*#og(Dlx{LdR%a4qVaiQlU{j2rF z%{(G%ucvWgxNChabdA^^E+rHUx7>L94^u|`fB5yxpBT0b1wVeC$8>-@#}kG#=M(J1 z+CwQ~4#W>KNK9VNm)wPde_rRmi{^Ka?K57y_y>KT<~uguWPSC_H+ueaY}7+1;Q3`< zP5ifM5{2nF)F)7$sJOZ?SK!pN;ceuire}CgJkgPA@{FG7_#Kc=hW9~!Bbei9{KCt( zahdAH5PpVpO37vz{j>QtB2UlzF7)lenE!)6$=~Ml!o2$>c_TS;-v%vIG6k;Oahx;P z8ovl-%)=l_JAh(66LKCDnLE8F6dc7!d4CBy9ty$#r~Vd`8Cb*k#I-4Cm{vL0M2?tZ zFD#5fQ5c_3033ADaa9YJ?Rzv`k|G!kYB(YK3~$}57yOD9O_WJO7+av3GrxM|Y7dcd z<0Ng4>a&XbPk2X0WR;$){~G6z;rXwTl=KX5=E%u4!Hf? zJ!%nE;(VU5g#yQHINGC7iu1j2Z0s5CGxM_qP5|KA+pb^z^AmO{2i{N87Br1VA6yC2 z_M*Y8V*cq1q8qv}sK!x6jmkYpFdy!OFa7o-U<*fo%A6G854rsHi!cxb%Llf9LhNz} zq36^whrRcZski4ew%8T6;19;r(7O9})+Y0hCPglFb8ml_Nb12UD>&+URN=(<)$x{l zk=NRu>NVUwkqc849F7Bao#aU#j3u%9D~>yIMG&~z7+Ls`L?qhUD1!1X>OGkealSXt zFkpc&4f7r-#RN@96}6Y=0n0Jk#-|B&!k4Y{9Hdz>uE=EpIz4qtnUfR>c*48MHxtu*n)OTb=A z;onW#2{ylCULw7g<)4f+Hn9Wg(Apm9JF zb^>ccorj=?Ua=G=tK0@psOy^8Yt8o0Z}>P3YAaDt&l}Pn>T(b2#%1nCn^E2gOi5cu zoB-HHmZ==h2VY=QMLj5^l7ZQ9u$j<}Vei1ssDN=eu(9aQT{=X2h7F)i&oZDL7)D~} z8=Oj7!@WKqzvAO8xEq{r2=@#md${{OxJ}l6vV_FlM7t)mT>$Ho5p9EWnI@ouyQ?*g zrUF-FDhWY}>xFi~ztE1HrO6&gf{w^cw4+W)x=)uv>oDxC4vN~TEzw?!j!#CrbAdUQ zZ|!{$YQB3$nk>1$1SJ%-R0G(8T1ZDHHfwY;!voSZ!3eCa3F$_p>2DBW&0D{)ADo+x z-8u+#64F7(BWJ4%oWSSX0JXZ1j_Yh>t?B)V{lBkgA>H92MWmzMK>F_aSWPiH@=~TA zD~g7&BE?-aDH(}!M$fV@R2>FrGu~sXx`SN_cGUS;2sSHk*%Y%O@yAz*3?oS~8}CZN z4(JZBX{jA33|)e~J|1B$I?e)n9*UV@&l}Mm>~i;bEPlh`lr1Dp&C&KClA6+ypl#e) zbJ+^%K)Wc<=0;Y+odI_rIv|<*1-b}k&vU9Le_kt`(Fl0c3j%;6q#JhU5%9jgdaebj zKRZ571Kijbf?6-^Z{p?dAqg*74lH^g8QTrSGZN0W1$Al&xN`08`5!R_;Mwy(Ms2(u zz}tv7yF|Gt*M=E%@*yf{-%WV)Zl5Vf!`{EnXDY7wy%u}^Z1^|_Zn)SdrGNA8djsyf zhomprmyk@j zX*LPCwE!tcg_GeaU~CRIBnyO_(Hl|W7v9{im;E@diHBvyKg>+?h<75~nD|2_FbVFm zzTK?ViWBVmvz`Svrlg7YOnPtNefO+Paf6`|@7(=R8%j*LG$6oo(ec({Ihte!Z05iq zc6j)1EC;r()3T}l*NC@1Z#2P8wEeI|EuL_v2)J9}k%79`67KbBS$~%E;7%dwhBKkv z!)h;HVWb~wTs5sk!Mf9%erQH*IN(mf;qD&1ao8fWQC&)X3VQYD=&4}f zX<3zZr>N7Ci%JWGhAh^zhZwlGcm?=cZ8zbbAEyDXr77?SOl=SN-P1C3@y<}M#tNzK zd`GKCyS)iGHzG{H1-GYygb8przrBN}NNwsnxRrT(!3)FR$(OT$EjlcDk1edlJnsW9Iyy*_aWk2V%-kFGD8S8@Y~heB5~RepIM_NDTuZWkMtq!wb}8Kc4A|2RwCx_+z4@ITej`Z3 zJVh=ZOqxosZ(yfqc~~kHl(B$o{|Pp)I>xDBfo&aC?&M#lRGpQHsUTz{=D!Orc8{@H{&tt(S&1<(Zg$9YhjENXBsOE;kI_np4uF@zTXe;$obvsoNTW)bf!~Vd!Z`791 zp|UWAX9AI#U~f527-BDTznCJ?R-X)_5E36UCxE?9{nqNAAK&qD7T9e9O|avQo6Wd; zHtHtRX?K;tWG@RciZt4FQ|Yjw9fh_hepw+qw+(C_jdK5wMwL$C(U`p&GgxyiU3s5F zjX=f$r})HAk^psDXMJli&W~^SI1B2IJ&J@nrCqz-!rjBM2x^W7plKrG0bZwxbrZ2u z_l@B?AJcHGy`7M^B%3*34e1@$ndf8LN?$wyIh_Paw$4ZO$>^E-L@P=XJ1_6-w$}Rm z_==Cy&^FKd!X(BX?Yrk=VavDA|E+|Z3oqy&XoNfJd`zmhCKsJ>f(K}OJ0Wi=cHt3O znvO^+r1Ibq>Eedm7`2Pk4Wa9ZG|mDgo(<5l43hf*Z972w^~dX3XpcB_qnT7ld$jK! zkvS`97tSuUZD3w2iGg-kg(NnITS<2I*D?I)1^;^_I!y4cU}*NZtU= z3E<|Mk^*l(pk1K#;w^Or_gc9#!5%-xX>d1s?#|PEq&?j6?iraFJ37xbz+Gj!F7teB zqL06JX7{b`scr zQ^jZ%(9X=YLpH&k6t&?=3$6cr$LGHfYi_TlI}_~raTeUTz7pRM9WZBZcp_C+T$#$l~As5x5Soe!`ZH~=vr%{Jw61%0@_BL7K z;2WGr2H{R=4fpz#G<=)|_sk)>;GXB%9`55&l7V1PxT*L}n&_o)3U$5QG)xlYn~m{s z<#DKFyi_?au*Nbo67xL^P&h!_{DyYH85#XwirP_Oo1`Vcoba6UIqB_vRQ$jQS+YEYMCB+d*ZltG~(=Oyjx`CsuKLaH9M zGp{yC#sg(PP@3Ty8MncfuB#1NKSDa0Vm2=DG)N%QHLj5^2S}G6J*4|*IXlEQ@?5 z$E12!EOz~Ib?IMG{7BiObz)l#9)-Y~Rlh(Fg6Klaq~b@WuR5kSiM9%a@Z+DkNJJ_N zwAWo4TT6B(*6Bw&3+)2K3U0@)H^46Uz~+5%Fg610Rurte2N1I;u$|YVq)Egs(RgHNK=8edW(ke0qZO^TJb@I6ud=~c%PY}<{j3VCu7=VZ!G%p zma0`nKs!)rf`v2jiB>B>JFP3MwpQy*tmnsRXrpWcv~Rf^S-p2QRt5$XrV(_99B4S3 zpgcr{D6CE(+Fs5!&-zN!T|ExJJtkggaMKTRzRH}sEOa|lg^O1j3ctC^U1;M3#zYpn zlUPsutnv0G*z4ml;NvW?Et}#kImKbl#WC$P-Iq}>E&586&V#_0CwewHY^tY_7qtyEb_%qGPDbP2$o5^;L7n_hRfH--x5sFn^F`hsV|)j- z{QX>k4<8Lc4dD|Ot8;2z!3$WYCwT`0>b2z0)*2l)4t|^kb(0-8ZoB;k)c4NE+;l$H zAgpWHh^y8C>`#HUH^ritvzzihKOMPinozSPhN|fs)=g`tU0E0vu(J|BisPgA#3E(L zFkua-t|nVwX4V3oiS+#MEUY1kC)T-_zQ_9B@yI^QP+~WsHcK9n6+%W=2=UpmsCeDn zjIes+AQq3;k*6b)mu2i%MffDmS7l`V8f9RU}L8zEF%)Z?&})VTAeeI z{`URbX<&OQlFQ!E$=bccF%^Mrl(r9ddc`hol>4xjP%$K{0u{ZI-9vk4?iN&uCE5g= z`;Q%LjKvps_i2na?B@z>g=;(kfEqw!=M73)YgVs?ITPlu=K(#k={l3f-Yd!4y^~Qz zA*>TkM4GZO+*cSu;J&YNh&RMk8iXOKhmfMHV~ifI1Mf2-=-X31Urz%YTkZ&UEqk#0y^}E#gyA_J_$jOzLNM8` za)`IXq8G8d=X|`x1M4lr#%D9Gxdhg1d1=dvItEb^h8$!rQLC`d%0jS2A#uzDSf^!E z-PYQiiS+z92kVN9fIcQk5(eUM|oIyJwDqstDky{sCY*s=OGUMXMojzRDU z66nq_YWamVjE4(1ViN7lt7DQyPcJ$gkw})vwVE&!3j}+ug}1dj=lj6NSzyO#s$t)- zFR?q~oK7`J#IA=(sEZ4oipP-TB~@>PY5LK)te8-DosH;Q3F}O(6SX;wy~Qe*PM0u5 zB_h=Hzy(NWBgK9I8y=5?ep`VJ%N5vbZO(+cfA6P(4K)qHzHv>zqs^IMqo)osMFkO# zU$7ydD3ZOX>aDOCq7MeU@#)anh{-UbO{}G0&>#o;>l16Ajp8hC(JUHj?t@sX7Z%y9 z;W>@EYF?YS*jk!1k#67HSy-oBmGsR!z+S;{N1M|%!49yoWEbXO*&By`7vYR> z1MG4KY#Yt}E78@Z$)L+YhQPKYu$|i7J7HPbi21xhNbaQpu^y_0<&D~?_UYd!gfLpS zu9Gpdvk*80!jX8u3^Vs7%?s56WU|0*FoWzY3*eYu zb9$}GnNZJ<)4;|68_^DJk9NFwHgbF&XbWs9njqgxLfaHSx`XYIdCF&A4+%kcbou~u z=QY7jn&E8#yKjzYtf^8+h1GOA8>xqY6*)X*(X$}fC9lC=pN$zGXMtUFNd$Y|F!CPk zduL;D+x$`$K#I3-J;65BREsylHl<~Buu+E|U)*j4_M?Ehu%g;E0hVBt$thG1JIQsG zi8E4wI*QhH(BBYmFTS0&M!Fv$oxZ2jkRG(-g57G|Bc1M@izO)w(cL9t217`D6tPwX zwVDON#oDo`4aPqnN=gVe&&8I0VLkr`SeuKgl8!~XU2|lQSO*1cTu-5(CDv`J4{2*T z&O|zXk7r?RHD_WCqq;rT_l`wobXX2Tq*d*R9zUAOshkia&iSoWN1OpHN+=$zJMS?j zxk4|R+K2$!^$MEVNL-7{EhE5L4KVN+FsF#pby-_+wY3gsq8z@5vv6*aR$LJVPGyht zy@Qd{>cF`vdT>Y}x(hq7Rz06pf77;a0v%|=+&Nx= zdG;mJ%Oi2({Np^NyT^_hVe1~~dnY1C)UkC8W<)vp~il=^R8ZizwuP=4Pjn)v`Faz;2?2ySX1kwnBR?#FH51y+GZACZ zt`aOb+ND2vIf`X(osSYMz#$Z9vjvuQ%tkgEfOcQs(AP?wiS_HZ*R#-$IqE7?a%g+B z?H<~`+Z!${d5aUY2@=>cLoM!`7+gi0>&7=3b-?*kk{ zi8Vq$7M0If#vVPe#x%nc>-7ow`fYq0r(r#k^1eKO(jC@!gg8w@olqK)$yxna9=wrY zCoiY61%@#wcdGCTAEPa##f~*&w%^c|MH|k??xITt>?EWqbPsHQWqzh7_LMO(V{zF; zZ)+LOgt>p~X9106lEk^>zQ?)T!I`7!*z>|s^z8mvFv(T~NnS~1|0}(f>Wb-b#1=W= zoc(%kjMsP|IC`=nw^TQuu%kCfXC6D6qV~ov%eW6X*7=orQFcN#)cW z!VReJ9gSKmK7$Wx{8;(X6+cjR#?Mb5*5Gqr5T`q(eY02O1--kWT(7*!6p0mdU+K7? z^DT(#(*WwI6&E@eHOtGM*OP|c0d-ktA!`ZF1lmr7x_QQz>b|Ml+P!m;Q|W-ZDS9w! z;xg*d13N+pbrRHVb3+rsl2-nmg(@k03?0)Xc_L(xPk{9;)!v zd6j8yB!}Gxz`EKStx+I_%ezBeS>FpoP;*}hiXcMz36E3Lx|ah|ua89gR=?HLa2`>i zI@;Ld-0vNU97u<(92y%esLAh4#{sm&sd+f5-+XG#fgVJ9ORXsi$@v$jYvqyXGi4h7 z554G*bctc(i|?sD(Oe$wfpzD+t@Sq(=kl$bh4kd261^MlajoAw6FG~{&ABL1gKXKo z^oes-B{g>^wEM97HqLG$w3{~f$`cWmWK_Hp%}Ahqrn<|CGDxHVA5b7o4;VzM`KF)< z&ryQ59{g`>_00r2f6HeX)?)T0_&Y^Q_HIYtk%1n=^?D85nV*%7&v^LN3 zSoj!H!YAJZ0qbF%iM)QJ$`9GKOaCGd#UP17Riek97gC)Lq%)fX2fcx{@@B%EzNOQE z)}GM`^uCNb?j4AnLC4BGRFpCX$#g9@j~ymasxF3@bZ>9#O|({&HIU|R%`6--^CZx` zdKYwIy33<_Q1p;E0RpJMZBBm7-5Zd`OI+30cbB#BW&$0*#j~J>Me~F@P&2s)``&>V z**C|~RVCP3K!Z-2uvR5Cd-(8*wGAoC`e5?W+%1lq@{AKOZ%`xh0PAh&Mkiub5Dz*L zStrM(bUrE&e{#sKm363I%Wfvp;Y6&5kgogPdOh4b5*kfcp$0+QLN%e* zrk}x+5B(KnkK1$;Z;Bx+ig%rgYM+T8n2>04Gq_3qn6YL*ff1C!l(J>BtviGZ%`W@+wX#yQf=Yuwc zaY7kzdm2L7D&Q8{UbF6=@v-DR=$R-$~doqh5$c zTS)*r85#tA?HSV7(whnO_pc{)V`%P?33i)Q*n@3%z-}zSC7FKY`sOWOJd6yLQDFii zw0${^8oKm89C8vS*t{>o*!GRum~@NZVHc9y)lvmjuP~>Wk_Gk zZzk5?zJ5CmZAiI@cD6m*cjPyQvyNypR7|wBf2MLzAsQr__|I&8VJ@!t<|XBQLk$9} zE_KSh%mnTF8BFeC<_TH0O>nf}(;va~lVG!03$4$5n_ynWQG%YAH}t;N-%P0Ie`kS> zt;_`5#(pbGPxnqpQy5{pC0EXW6KjMKfUSxDoW*PnsW+9h0d`|LV(u8nDwJt+=>|4< zo;lnFtV0vn#5y_DrV|oHg2=f%K)p5%>1zqj#QF6+s8bNFZKhFsr0*S&+-d~-nV=w| z^)prWduVA#1(DPpYF|vaWqxW8@E)1`Dvs7?e+PAC%+5ZOO7}ela};QaH4CG|I2$N_YCSQU9*YoxYv?;nLRUQLY(*xf7FgDu=3BmN;1 z5n|1KY!E&m3IkwnYQz3wpGs{qO|_z%Wf9sXDf26VVFWsdaJ134uZ1|1*!^oi4es8R z`PF%2+reE97aYIP(2#I*gdc0Gvf@bcI6|Xq&2zrg>^4FjbQQ|66f zk9N6-cGL~BjqFR5l=?+XC_!SE6i6@%4YY?WxDcfQps52kh&43`D2&j)Or{vrjb!O7 zxd2!PBp<0S*#k*UirRRhxVIhcf|1tLUMq1X)b`cR0=tIR1$It1z`l1xCRQnu50RQX zHKXiS1-92#y$9CiuF${XMVZw7b^97?}+4NKAvuw=erCidK4)VsixuCNENRGs0^%) zQKb6n!5+EV5wdA49wOEO)m8IpN#u5${}z4WgtcYG>=|IN+t~>9dO<{At8gaN6|~bp_C(`7^hHuHFa~Sw=MT3s5Mhi-7k*K0jxJ2gt`*y2zlA&M^a@4JfsiF^mpf$ zW!%-V6(j?0%wPwu&s)L0mf}oar?2@exLaw;{Zbt_;6A=SR*s@4_+`oQx+q@5$zFp( z+O_SyuwXVwLY=a__%c&t>||`jnnOy+{+!rZI}Yengv%w7 z4j!%p?=!8ocTi{C)2(d&iRCGII+$qmNvQd_0P2}cVheVCj=T!3kX~zXCeHC|JPm1U zO8R1pJJ~|Nqs6Hb6eP@gx=Afhzd}C>&ULD|n$!;KZoA3>4q%-KHtiS)HZL@dTYVht zI(Rr{H()iEpt@x6@cR_l2wkwIn^_BX1vh}bb`I%lEzX2Gd<|!TJup5`u;;0=2ixv| z-AsogjoPzo&H%O&k*3;e@`hNC@0NE2tw0`Zwg_t;jV^c^)$^%3+u>8udaI4@HuapY{c zDyzvnTx*#`@{}(Y)`DUGu3UgO7X_B;CsaQgS!sny?u_4D z>q)TJz9D_B$eC#WI16oTnW{?qLfJQH`n{8p1N&HikrYF;W(F-t#gI<*LaI*dP=DRh zlP@G4zFQVx^K3M#7aVN75ff~eFJJ-9rm;-4IhN(qXyBl?}F_<{aiE|1}6{cR8vt2s*UcqbdMtH85DPmqw8Cx z`!z2FxGc6O|JIzm|ehIQt6bae#-%w9IB zL!#BEu%_6(@!0TvWf7|w)>to($I0ujuiwtXI;HAh$8-bid&eWqXaH1^DGPIGI9Vc$ zAR$mH;@z<3H4zH7lj|g5?XIE-Q|0-%G!|D`Tg?ScgX$WFqL3{KL!#y6OXO6va!z~V z3T5PdF|KbjYh})4_1D+0r(q3wGqCQqU!XL?y#rFi=+xfHz=Gp4MWt4Nt}qh}wEaZd&4V;Dp5BkO!bU$W9R3ma09s2Dk`_6*CaL-mjb zG535)&vEqF_!mu$`>4+vv|$sq+~~cq#u0e<{ADEAaPeD0y*?XXU&re>3+e*XJ3!rf zx&igQ!%_3-Y?x>)Iw@Qmj6~>`C#0R!=B(~2KBdG=B)bdM-;`2y zHU_4B(fn*wGYsN@_B3e85V;t31@>B(Gr{h!{WP#q@Fm!JuMQsZ-r1;`^g^dL$-b2K z88eZ{ol!4tkx#T$f2yAMCFa=K%)#d2SlTbFV?!QNUAuscg@>qg4b{&^wZa>v~K5NJ#%9veKzV1tnVF-Qu%~ z_9C@9v2HvX3$+Uy*qIUA19leFM7sFNSm@h~0qUp04efZG4oJN|7yGNdw$p&_RXErE zwv&zwdH~5@Qw^$zNLbU=dk4_+9?HIX7jy*BQRgDSopBlM^t!qI zMc!qad={cZU2!6C%db5#qnTKzwR^}|3vwo<>#LoGwUy#vhkgU>#t}F59U^~ zF6q?+NcNg4dSJEi)Z%`npcP4`s|*X+R&}$1jJxWntVM1jMSoexqkmo=jg3!>TrROE zoW#ba$f!LYE3nr`qfM~uSztFb{Q!2gQukmVAB{y?1lfLTcw?y0hY@F*QA@PS{h464 zgKb$I$E0pSjvlTX!*#ayj@cZLu5oi#R1lGLVcvN~ZNs6E<;p)ef)JVctk7PIawgX0 zwVZ}F2t%|d7j|gh5#8B)&x^thtnCig z4TXHJ!c`dIj0UjwFj7@kjXn|9QvQ`;!_kk|M)z}X#!_F7?aCaW^O>@kG( zY;B7o(ov^li%LUs;^~;&EzYq{U8f^6n9eEb;wB4_X5uf(WFeSW`aEDmUD&z*Ai{%t z3HDl!Gr^AMfjvrA%9m$;w+Fl213M@cu}$n06tJ_05EItk1Zz<-qpD1cDbpiHTi}M=C&)0Ao*y!aE>`?Y#*Lx?U=X}i+LTXUXmpp_t z6;z}5z_yG}?Z&KD0V{w!?|6?%F}{PkqK#f$vXFIWlwu)dPp|3R2RY#k-r9;9nb<6m zUZ0C${1`vaLOLeZLXw3WK;JtS3$+7<>&|2vs|p{wQ(S{asVYqoq5X0-U-x{EdyVIp zPTm5mjMb5TI1;%w8pf5XYAU>f@%MrvNMemI3Fi4njb0=$GFe~E*D{>(bHe=NG?*bU zC(g0$8ZyJ~9g2*!smSk*w$qB!$)iVCB{ljqm?U|fdtT`$Oqkt;nH^xx-oC+X9FZ=* zo-hrbh+Zb#K64w9#xlvbnYbd#%EmwEjB2 zo(8r>#p=-ayOKt`cPPr-s%oMX^}MFUnzBf&zRe2O4z?+;9z_{Qk@-+`qc|3&DrOU& z-X;<%jpB-X#B0sGJ`JrtiM3i^m7DHZyav|moB3LUGg>l~!mJkkn*Kx+n!i`0N~ z)dH&!lz*@mIKFJ>V&S_VEwr%5>B z?5y^kY{7HABJZIXJ=e=zHb3e+oH-et-C{v7Cn{D)^aD|Td@wATD8rc&O0@Nbe66~f zD9^{@43vjyn_V55-oSg156(l5vGiw*f*Umdq?cAC-i;z`hOJoma3TlV1!0-U|tMrgF;Mn04+F__NsczaG032KJuD=Xrxg?qoagQuw3b<@ey_6?pA& z@bUq8YjMko;1w7_%`Dpr{4D!EF*tIYA$XLLz+DYbZQ4Jz`RJ)Fzw4=8i&f48FBd4# zX1Z)x-DW#O%=umQJIu2`_t>tOXYax5cfnh0PtF5xUO`c(zzL24;hi30epl%s=JgBa zjX%%4@-gP+PdG0kHFLF1@aUkB_uxHtV6;0i?}B;pqd~9Tjd{6$26(tm1H6Rkf*ar+ z=|e8TTVyI?UL3!u7e5^I+Fjw7pVbepmMn413SK@}7D9 zKJ)UDc^^D&>n9u-xuc;uCKp<}ovKv(EvZWWQwPQ)=zYeC!AYALdkS1x5cZO*o+}KZ%$$wnu${iVYRT2K6!*_^lNY=M`%nz_0v=!O#9vRBK{~ z`o{8%rrdY1KgX^Ap0{LQKpPJh(At9qw6zxEL`vVq???;r?+m~4pMugU7pN9yUSH7< zAN#!iNHYOTFX==oExK^l(XiV5ctM_jwVsY^#j+ficKY{?C&Atw;r4J$NP5m?psp+dPYBhpZ$lyum9(W z(Y;AOSYg7G%kJ2WhuVsH=4Kp$9AP&Z=SON5H{7bw1nPp0u}nLgp)Bl=Xn zi6d(FShsm@wr!j5KR?mO`Pnpswt7cm_=LOR1MrpqDDeF!xf(WYfvEv%({bYs>4$F! z)WJpbc^H?W2(5LTf$)OD_8`^SX66TnmxMP&wtl&``G(*$EZ8GFfse3DLyYP3k6?b$ zpSuc2*pI;P@8AuIpK&p~hu)zK;@=f||4FWe9cE89Gc5Q5rMiDZ?wy$J{fSxrpi*eu z$N|g8YELnK#;x!kd@1}#fiL~%xD>{UwZ)ll$Ovg4VGbo5Z`W`7J7d8*W$;BP@*a{| zzQF2(tO>z#$-2UjbR4W_ALXN(7VK=c|eTGu*A8&~tq* zWB#fK&P^M1icbdF`}4e3)%pbQ9UP4POT};p9P7$+gKtZIDlj;EwgzoIY!m0NF{k;| z?8`i2GbQ&o_44!kUtSc`YyLLQu^TCsvLemA9--b!xP6Ix| zJb4F0%n2nYRG+Ny?Ez5p{eih+FMt`VIE^zPfH_q17>-DQdEV~vsVB-9L3wJOdm@2y zT$6b%I+-Yc9q}xbAr)DJeCWVcI5@2>28|fQD;HS>h@C05>71|`og2>yaRJ1Tm>Y(l z2=Tlm0%G{O)J&ft!+iS$9P2kH{DEt&%HX7`@sR<XJ~+7q&+v(o#!ho_QKFGH?xh#g+PQk;_)7hD9>2JwO|n_W0zyilqDH_ z$^delr(+9zI|_YJ2r^8CX+Z;!;R(hDQM|7uY^>0Q2bI&w}|lbvrf{DcBg`esNZrtYLJybDlPa zNZ$*3HdFD26ghtJkkQI8aS%Il-{}%#ONSVr<8U!S!XBRkjE@h$&Hy2sr>ul{MqA$#KQS(h7cc}FmGjx8 zas`#)jcu52o>Wc@s59H`6ZMHQ?m;%VjH<$0h+!G)tmYT23>XcIezfG&5?xw)rq3u) zUK)+hpT`bSemx82TFOBlXUB)%keMN@EG!Y_nO6;Z?*U}ZVt;9q*P~#Okx-4)1}jpP7q}X_!=%R5h!Ped0t$KJm*aQlhFEQ9)4}PDsv{~M~B}` zc+P?u^G$1*+u`9?af9}u|H%B<@Sbr8?tD23NZY^pH3o1-Bp}WaIO9HJkqZ|M>onps zwamnMem}q&jusPVqC5i!P{s)gl;_0*-cpDykvU(7Jx?OaZ5@_dIYjxK>3@9NIK$T{ zAD(^{7N?mr=$$_5GbJO+qhz9ti~v5^Gb0$--PMTlyoNAqc2ys0N;`xIdq;dW5?5l|S6fXc)^C_FHO2C->Z2jbS|YK z%%Lq|9^nw?lS*>HpdDm#6OY0kV7rf%X$dH!uJjA$8HZ8Rj0GsC!UQ7>hUGHE zUKW|>Jx;_P06DVgY4g!3NqsE1LggoHDPv`EBEG&}uiZq_m>`FlaAJ&YXqm+~fM9+nnP|02cP5ZIl>4(@Wcej%XM}0$4Lhumv&B}2+3|N`2gf{2jn4)QOCh0q+y>{ z<)-Gr8^$v)!GlcmD~UCLj1BS#6qALhxy{5GK?J}ees`u6xS$jjhvvUy$)q-djO$5; zgXc5ep$Y4d-7P>~`s~dg#|}aMaTdsjNn8rYM_*$jL=IR@_vuKKv6*M0%tc`c5E(JE z`h{c?yA^H##IruLt_sUg1IoN!CoDPTB_t(Q=NfamNG{{;0L&B0A}^=;uVFntFmuWS z0Wzl*%xh81ggLkGXTfa9c9$^6waoPaVG6yic1S<(QrO?x(2q-9)&n!TXCN0?!~KV-WMn3qJ@=FtwCekmRweuLd7zzm;sT=fi9^oD&?fEgJ8N;(~BR`IgueB39@32+4(uU9Om zDS+8P?TFlh)uBLY_s-L=%==0)TJKGm%RC4$Nz)lL(oFJ3`Jzg}{A6tn^ZNMfIkj_& zMU_>Abt0G2;qeDHvbTzRh;Gz56Xrf+$SB zvG6|0pur1v;z~!OV=HhDg%|8H)9%8m_A{OaT!SjjXMyuNmrHAN65>40!nu}o0Q2Di z$Xgpx4k%O%*bhEq8(3ozF3$v^V248o$k}ftwF6{>G$}CULvfQB<9Z#r-vs64&a5s5 zmW^;gbLyw;Zj*&b6#vrfY zG#*$P!cOL*5>Gd=daz#E(4|v)FbFY<9Z2pb11rQ_tXlAT9aXd+u=fPWHdw#)4GIw8#GREoe9Y_(p(e5`PpFc0oQ!>w)r=7a}!Ks~|J2q5Qq zE_PaZ3*-WjaS{S#7By#f)__$gu zqnQIBzfHK%py|i!iL+L!mNL*7s98FJOx8+&k&PKM2WGgLfm)@(U|jk-7$1mSfN?ns z<6+|>OnKfNa#dI+`vX-oIq8q7rG+@P)Q}3T zhGY%uEuc&lP2*J%m5elia;DS~TNZibtxb_O>kjcWv>0@5h3X8@lZnmOJbzqaJ!E+=6WNe0}6496uC+5dF zL>4|Pxg5GKn}a_Jla3|I2d7^8(c)Ps_jJ(cOUE*o`O|Z;-lQv;jC3|(103sObz;tK z%1 zU}F^ySMepp>yytW#OIVcp$U!$a;;?H4#WdzKi1iImfa?muy=Jn&0gT;X6#G}ii_eV zk+F@B2VXMEX2;Ae8rt0ixpf8$T&N+pXp}5RMvHks5#!1>32K`YW25dd(>*+9=87wj z4>T?lWczs*$a%Db^3buy#fBCN#Xzahz@o?ml-o=VIy=rlIn6)okrA|T7c^kT4k1Y1 zkQNtS{955f4JE6_1f$O90WEdLWNd~nh4oUvoDF}38l0MgXqK1GlMm~)mnho-%HQq# zStwgSP-~@QjSIl?aFP95J1L5N3@W>!q>sLG2FNh>C&>21GA0i`>MDZgc(v? zgVs3%=EmL#@7IV{Gh$TU=OKHHlDOouzEU4ZT8g-;kfXn+{S2`tJfoEJ$!7|#HTVoxqC zP$tQSP}eNFS##Ujb3@&<^ZKllFQ}~-*nNtr17qmB=8q6LBE(MykWuhkqP#x&N_w@E z>YQjeA9eYT1unQjXa~DR3JcM+NJzOVYHnEDiw!x{%$n(9J@LUs8NNqUzyM@)=Kyjb z$e5PE-Mg@MtTqT%Fqh0VXaE^82Cdd-W`NMvfYvM!&9J+~_(0tPjN4fl=W(FbO2_IJ zH02^!VHG6Gm?@YT4;qnnSvElEQ#0q3qPo?SyU2W1KZE!f{pmil3(YWI1n-e<9-NmISb%n2O_O}EN!7bk5)24y#hB=d(J4&6JdCbVCrfc^8amFGj`Vw7gX3 z18Ga1AfHr&LlTh20m#RvUT01)K@O3cLJ&8jR8dfLL(s*BGUIe^>AF-CWMt21X3X;n zXw7^9-M&H|8zV&xyx%3)vYnBTiW1W?g?{4 znI4jq33FnNtY-?nIf|X7ebYr`KHYreJKIKs!4-9kmNgrq9!E>MRnvQb4ZSF_EQT$O z%lL3yPFyiG1&tWS-Heq^+)|b>ug^VPyG|;@O+JN#{$4&-w(zQ)0dr*qxy>wgqCC(H zfoz%QUTwxyPgycloY1rpSmJvuY3WhhqcIZE0q_jL8J&2J*3sMrY@|{s`!ytH z0F0V2G!~KJVK7#(K#ZlzOe}bt#=llE2zQdmR2m>>Ns3x|=70%6hQ9_=3%ClbKt9m3 zq-6ESSs+_KkZ0v%LCZuEZPORj8yl6Y^9G$Yafa$~<+0Z~TmMbD#SAJFrb#z8Kodbm ztD`BDV!XF8xtL)ZC@0l5BPv6zG7;sdVYC^$A~pb-NYCr;f!KU=eTp=sRwNk4l%V*XQ-xEiveSd?XkVYiwG{%g^3Dzor!9Nt{&M$ z!K)yMWsZTr2*C)ZRj~3J2;&;KM7SIx{Piq^=f(CQe>*mC>6N`ea-z;zAzzDyA8vi0+!g2a1+?YpSOd zH!BQRETW7&omwCxAC*T-ovS;KnJQc!h)NJ2O0id{&zI0(+r?cvT4wI(5nn43yF)TF(ofgvK7$|Jyp&! zyTWh;i&qiK&>Kh{%S~XOIr1~cuK4SMH8h4$MM*?sxQkTg!4)Mv1b8Y?6zNK6rvf}(+<~#W9j@8@0{@4Hh zFW*1Q>kScs9z%!-Gk@5>Pj)9Dl-7Y3whBaz<0B*#g@if}CxiqHhan{7rkKzf6mVE2 zIXfXJ)OiI==r}*Y2^gE63~G2@xsg2#1chKm_qbASzb~I)IYtc8asY9hm!SdkJr zyztBwa|E###Z_dZ6GTe_5i)t)mjN*K}3nSRj zMqvK!yjJyO`vgNVV3=?ZO{} zVgSJkj-ydU#1srfjEpyE+QU`}pES@Ca{jHP3qke%S)sl`-Pk2i6P|l=)h%R7-l$5$nP1=;l`Fo#EN_% zukw!zO=~a>2KE7=D7+7coTO0fsXP zh=<+& z2Jx`RMk1!}M6|Ld;wLzH0mOpT5Fpyf`+(TP9T1V;07SH&0;ORuykJIxh+}(#hz?|`89x=5V%Z-fJr9xEYBSbc~jYi0Yg-~1Vd=D2tPba>$RQ9Xf*8c($r}Kn7lP<> z?g(OrQ3w_+ZV(-64^uM(am?H!jP3_0uZUn1fIjR~X3;ILkjgswgJVZzQz3&xUzq>` zPf)*lrp!uZSb`2}0EoFpEV@hp!E=K-1-ETtG+f}q#SPQguuA+&o#XcdTvmyD34aOZ zg!XDe(bYA6pjrif7OY_UK*3n3Fn#Y> zara6zVvhC>{$UewMLzUmqW=M!Dj?VxPocut53e1B-qsL;U(`_)0X1CSqo^Q0pdm#I zFpUM6sPWkMk}@*ZfLJpDh~^beBf$?OGdXh?OwKWN(L{{^W5r|met>|A@S!ak@YU|6 z?%~U8_|OL9cuk-gOC=9aqGTdia@e1N>s|zQ(cTYc2XY|cZ}vO^G0_76&GQzB&{xWw zaqnK=NRERPBIw9tdeM*#CB_oqOSD^DD%DfH;cyfxa5}<>J~Nzjb$AKb-tM z&MaIfAVb5k?|A~2DEwR&euyfNE5w1=@#}w-02}#0oR|F(TbPeY_(kwJBZnjW=3D~N zet;7S`1vq)aO#)j9>3)sBBogkJ+(XBm-0I>bo#(VKu64UAbKJVm0K)FG!gdW5~IYd zLwHi(<1ueh9On0U9AvtbEsYg8=OK#W2k4%Np!&DF@oU$@zd(wE4s{z_wov|1_0sw2cL+yQ((E>|yP_F@iKD|lpKCt~y1%9x8ARU`k zg7f#x+rMq$ppeD_uMn5?)^0xF8PgANCJ{gQTfooxw$CJ-Fh~qH8bnAH(`x;%M4x&F zfEENG7*6^MWHuU^!J_67q`HtdRN2_EfdZU8V*z?CKyb^T!~$MI064!w(hm?K0l#Dc zg+^w3_h;a`)Sj3kb`l^#G)(+35DUkbVTd2uW)5LKkiCF44kWSBIO0_DKq3+A(QlcU zIGn_!O6gdC^Dfeu>JQ+4va2ScbuqEtd*rU?#;f>wBPHLS z%82;~h>8ec8#_41vOfXW<#qINLPynsf&uNbL=ZA84mCn?Ji(+zb^)ZVKqeP9{IHUvtOA)(`h)oI9KIhQ76N<~1xMi5gZzC8 zVO>*?>>%j)L9PPpiuj?S_yG}}@bgaiMebf^xElEjIa|=Q6nPe=x4sTG%b_9wZ!nQT zneffWbYuAe)*#}C5f$JUa@+gQ>vDReMx$pVLD3%_G&J;}A^QiY9gOuCm_kU#FE)(6 zWD34Lu-6UvS>50(f-krP_!JJK)IKf5#rgw$K!6W-CeX8TkUm(~(ZPA*(bwxMQVFb; znc|D8C6m>-1@ydjuR@a^{vz_xPLABkt!&AckOUs&upmPn26%Hx(pZ0h=7-=xlnvC2%X%}yKF)wXw=*F|(xFK}^yaD$Uo=&NqLa$68~gbs2LgpPTX zZZ2S#0+XufgY)37+@SMXcbI^I*mN1O%p&X3et?pP(A9)HmlQ<%GjClz57c|%d3KjM zF9qztCW&$*h^N4tO{WC?zJxEavm@h{Kw~jq>UD#-wB=kNV!}>_2%-7f1YezVpvCqB zR6BsrN6UiG?GpxRT{%ojS|O{Q!>+(WB;iMG4%cJ$iACo?-Br+BR9GZbyMtEU2iBi`hMZ zmzeQOh2d9g0MFDbjYBU}&tYJfm`DJva>p)(f!%zzI=LTU#{qVh#>Opv-*lDMHFMC_ zpi>##kUeBUbOpi-UZT+ce!z@XW7A7iM}8Vz(o#{VxO#OF`(R zX`&$cAK<+Kdch;l3vxEwFT8cN+=i!IJ;z0`FF{lvDvTSfdD=+z*v zjLwHQPY*OrK##Tc_&5>}bxsqT{14FC5Io2>1H69l?+?9ovD}m`;c+A(WWdL*DSlVL zm)M-E$cu7UBIMS@oafyP$`)L}wuNaPwQ*1apiCEq-JDfE`5$1a0d^5nS-`IQr2Ca; zZjD`l+rcpea2gXjwDZ7yGeH;GorZEQpv#Q8GKgA@A`?iZLNGkBm2V)Ig51-qlX{@#5_e6 zaL?+SiDD)tv`4Gn3LP9I+9K9bPmjQYSCo<4GnBz+83b{B~jnl`OZ2$)SW>a%$ zBzk@nV>P4XFTHj(Deom7QO#MNt zdEi&`9KRnRg8_ayqGPt>cQn0~*5&bj7vP)Ori&1i@Fg|IBL)F{P`l?SC><7XRMH2j zl)?2ZJ?6|2(6;Sehm9W9aP;u*{KS)FGp`Zej;1;xb^Dv6u?jubf6S3pX_hrnimj zqLvFrcfSwoW0mQ^AXn?qS244CB!WDv7gk5g?zUGu{ja_OSp7ITq zHO^8d5=8^Ms!eMG&qSq=;i7<3qF{nuBS?vJ?hw`+;5qHuf)mlkSVCF}9H;aHTq*#s zfscpi9Smlrburx04S~;lClWS*msQ|N!p>D;ru<)*K-{zc`xFD%Dds ztBE|maSWK?`T3ETet;r{;8iq+LBaZ9CM&H=;jVki{jveG9BPtPOV2kI$UGyB0tP|i zm$h*lA&K|k6C-+sE6SMVgkxZzw5#``6#*|1DEeMEHJC229W_DrtYCugmP{e?jID ztWy}C1C21*yNq)@XdWqdsNLB;TKwnuMVrai!g_oqADO937^Im%h!_j8XL9B*_?Z)K8z=YFdH+TeB5JX9{TJ(Uy2H8Iaax{hh6T#8h!*srdI{ zsQ|W8%0Gga?I^vn)@Q&~;L-o1bFbjsL&e6ZmM(H{Hj*{+q8DcOsEe2ANAW#s=Nh$7Ky)Ibk~A3JzG7G|n{CuXX^0Yaz|;mn>(Cao17j?Glw z&6Ia$vYDC6m(5f^m}z08Y3IV<+NtNj#ol~3QaWsAO6ztA#zXAJM!TG24KJV?>RGhM zWW3$Mg@SKU!X2u7@g9yY(@T($cw`eqB`rB77@;-FQC+A)*!!#cxr&4c&%9jj^n=Mx zl2(TLyKyDR#$3ChP}<-3t)w-30e4gR)^7UWdFU1HrutzwwX@yS{%(8-B!;0k3c~m! zpS+#<5aY;0VB)B^_I5Wx8-;c<3Sdz-?@r=EXo<@o&$B=6Pd)oT<#K9Y^C5mcW&iu% zw57m`NpBD0KPBx0&viq|_q@F{@%G}Cj(PiI-vr~D5e>^c6CT?+qn z`}gCY_nyVrW3f~=gTwU2BhQ1Ge;rIW|NUdog+A!kmh9W%5Sge%#&)@wIAAhxXv)vSxZw>eV``17IyZ;;D|Mjo`?|=RE^5@ro`Ah#x z^Z)%Y!!rZcJ^=gt$DjYx1AgrP@DF@%JpWJnE#3~T?b~j^pRk!0bh&hBD~d^JcSGeh zxi-j)FuIc|5&Vwqr>Dck9=>*WwAb|0M59MG33<2JLp@mRF&-}Vn5T;!yj#c-1#s^) z4%WfZGcpP$)#<%)iViIj7&pTsm5#_m3Z^bxE1a0;Kn%*blCbR82FVZ;ZXXv!S%_BK zto{@fyrn-}yvAP&-y@ZqBkQ9bQ~ciQkM(=2KkboDM@Z*mr1KQ%giTH%ar76nRfjDk z7-nwhi-!l4K+CJ?Tktn7qj91>o7q5ug81O8uP)ZKnIZp zAu!p1J<`-7PssxuAb)JGRd8(j;|zEdS4I?nTFiE*?|INSP2g+ags*-RzP5+29l%%I zA$-Lx;LDOm0}X!7Km)B9>PL=h)I%-u5x{h$BV}hWV)4gY5g4kf4`k`U`M}cyi?X}; zRV5@e_I-T4u}6@uQU7v%4pB6kb=tqT6J=^Zx~6VJepBM--xDgJww zDeilgDSl^};*Tt2$Ck0VWuyt;EtAPmKq?L>HL|lFyqpPsVCkD0#ZT~s6ZE%L4ar?F zcO-So7{?&k(`9jd{`T5Tc~?i;j(|hSkNTC2`26;D?tV0;>n-uc%4Ui2%KFh}ddu-u_#^)9 zfv=qy;G0)u`Q{Z__E%&%x+2T*6!2(8D^!r9lFm3kqYd3FKyxvV1uDR zB8(sucAgDV8B=}euhPT&>y2$KT+vmGyP7BUeDlZK(!mFm>o@Pl`pvts?(fEWbT`)H zyRk0rM$G2*6U7-@Cg^oUv{WE9|19?bouTLeWut`%l;G6qMH4{2d6m^~US;*;RW=Q? zYjn3^cD=psj>Bvs;x)~o`ZhEA; z$YE2&(HO(RDXQE1^MBt`pn7Bql-W@H>-g{U`#O%7Yi~5?zOXl%i?@5DIXmAQErWDzG%{qjJItG_U)~!W zB^7|_=HX~}H1Ga~cSrk}_uu|0?srEsceFd2**r~xW`)*w2L-v`$^p-tdf0nO;GY@X zadkolcAWZNYnz=0iR&43Q+VNc-B{Wmzjd;(+s%~E`ND1%zqhc1ePv<4k6LWs7Qf)7 z6m06gu0bteyEDH<_vLl{hF$pZ>EW7=f6Fzm7I!tT77uD(EgjapTAJ6qpa=nD1W;jk z@@#r^U?BVP{I(p?6i*cdGQQ#^4|>gHb>{Dts#^M9sjB6#l&V_(RH|zE3#BRoQu9FM z;Mgtpjbrx%njaqym+0S4r{s}I_L+-zl(}ffnTxi{Tx5zwy_Cyk9TXzy4cWq5Zf29OMP(_d#C&$D<&x|8t$!QZXL`f#*bGFLt1^jqUvi zMQWXrWGH{?1?0m%T%Ez$w_>{f`ftT_{q^-Lt25ljbp0RP7i#?gBjka`!SW1eG2P(8 z!36#XDBnLlT$|ymjYri3ptvhOX??7p%@bNsYKvwh(J1tzs_ptRi*&1)HA(%5^l z6@SDt-##5K)ABbQTlL@GcFVNHgJoLX;WDk$*eW?_*x#o}eQk7=1on|yDDQWCPV5o^ zwfKWx@t~JXR%mnITcI_6Z-qAd$_j1y(+X|pF%scG!;}Eokv|2$?ml#E=zR2rNY)L> zj}J$y)Bi1Z2KL|nbClHoX&)s8f7lt=|2fWAXXHq63Yq*lg{5|6P<(w~o`#l*Ch;k~ z;z2JtUY`DY{e1oR`&N+D-~S6iQh(KLkktPne_?q><)#G4PMtjXvwpt9J|!HWAFymU z4~Gl%%iVnTZ?BuL|Mq#eKo7bYfI&AkxJ`3zAw;}7{)8Pch6RaZGkp64L{@| zc&JxA@FkN4`n_(0{`=MMEztCp5XoovX@RCMgh<3A+5lu1#rJoOUtOR#SblssT%bAs zEuWxy7b5v`5F+_<7$W&H50M}>ibkdeHmS-U(kjA~_b7gA97ColOYa6RdeEyTi?n_( zQu6hCk&>@piIjZ%6e;=kg-D5*U~WgC?AB<@XVu#JUakEA=grgMDs6wm^{-%V7c7}O z2$swp221AW!4fEN%-<*K4cDzpDHJq-Qs+wCQjyIT!0-m@Tl$OZ~Jexra8h2%wq3klQf8=l{++|S85qhqSu%0c9?qo__;EW ziSVPh8vhJqpf%ehTM9Z%yyOj0y~+FMf80@lyPsXPzPDFlC*e4Yv|LRX_rwXN{_Y+m zP%9>oN8MK{-seCP`u?M!rwk{{N&dZ?@V<~9?E#lsL8^~)B&2L7Vb~qNF4~-%&;M~c z?bh*~(lhlS`nUVvNes6A-(Oz;>tFuet?~c)*I)ki&;RwmgU(2et`?){OFukPt{*>r z`0xq{)1p&5R&-fzdl!EMen9^t;0pZZKmVs+|DwO;zy1&Zur22Q?fp}}9xv+OupL6% zU;H2c|M2I3`CtBX_c2}{^b7o_*XKX~?qB%J zpa0{(yua+>|J8rRfA|mo-~Z?TgMWHA3n2Rs|L))YyQOiWE6IUVm^EW_(aOh9A3nUI zAuZ}c_tVH<>OUKYnab}N=dC)(@getpwAC07U82ux%~7kz8_Aqvb6_n*=Um8?Y#X0EeE6zI4HL{k;>nSgQ~A5vG_T(@Osh3_7Y0X~PtG;C*&jZA`0$GE?{pU$)BZJe zAIWTIo$q=BrLK>(P>cJ!x^Vu-?^>s|ZV6p24mrm`r10_cx7QdY_u72|8Da~&8>fEP zAE+n(Kr6nK|0~}(trky>SO<1-*W=6G=i5tq;EORmY#tDzY-zO**7>eKQ1jY4)+6%v zf8{%;j;q$lG1lPby7w%KvNY&L{G}^xF9JIA_;QB#Y(%+MP{+9#$d`Pf{uMBnc zSA9ybhOZ2Dq((hUkcnbA`%^A3tYxWGtR0*b;m)k@U`#U>BS{`2VdSYx2DjK9+)hyXWr>pl(WXYU?*T;MI-=cp1~BFoEt_exQ1Vw$cLs^_xL zvGlmxB2#-e4}aVl>8cQ@bGC=+LI@fsah-^b05WzXC5bi{G0s%uOho@ug190EyO`sI zc#mTnHU26{Cuh6ER5Q3KN~NqNq{;BrhF@O-7YX#`URs#}KP;1EKZpL;f$t)S*alL| zf4U3Od3^BT?@nYw=)Z6U0RpMxyWQ@Y6Qcn^g$HyR1@eKyjSfzSXfa>Hib)1t1j42b;ieL(s$cXX)Kb&ta4mMsQ# zLN`RUbeBL};i&-Y3S^cnq1tnw1Nnxu2#}4t8$9SJx~w6OKLI%g+`$!5&^hhn9`khp zcXVa=Cbe5kc5@eQ)G1)(KrsYjc6l2zI{x$PU;odyhEWI5+F>5O?0eM0TCIy~Ld6|} z;2iCqv4R|u{@}vUAq19^ik`|4!R%Q%hRoswi%s2!yn`xNV!vwc-+*ce&cR1?ypcCa zBJ_9G9s@K9JbH=yF_?JtfxQ50y|L2(N#@{|sSD%z)Mi+mS5=a+IJw1k^*e zg515WsdL1+NVcqdoc5)p+XtulR0v+c%_S9?euKt~M>aG+Y`8XIUYgLZGNC!Jo4|z8 z?OlkqPT4SFs<5E<>5f;Q!GaiR^wd9zlq&O5(?gp9_8@!-FG46MQ6$WRb(1BIJM~K) zj6{+EL18geo$K%l`kVAk8X@s$_oC=Z&=L^~TD;5B|03~FD|b*^lEs?8eHf#b&}O0m zv*^4a{sIFglK~?VYScnZOA9Wf(td@f<}b5g0mUvXSa$W`5bP@k?2exm{(-Qd(SpSa zEr@&E)0zQ3U$k5=2T8 zw_ta6u>}KyN=x*fDvQDJrC>Em**$c(vswQ1Boi7IN+z8!344YKOOpvHyh9U)wF%?W zgzhR624DsgHgA4!!~6vs#*T@JTp4o%eV+u>wfYqUZe-Yy_M|jc!#QOUxBSvq!3|l_~cSej~abKd3Bo@?0 z3=RSMV}jrjZ=`NSUui^MhW+TK5j}O2pg28r;kYaNf!eeYaTfOjGRU=X5j>l&$%w8m z1SoSdaLE)9>PAG+1|w1(r;%IM$a&RYf!*D$2C22%C)qF-vJruAm|9s;HViT$9d+KT zb!o%6a%g_l*UM}O$y3^}+TH_?UvX#xuEPwWXxPwVh7F@bGfb;=XlSsM`l5m$q(cj2pm0?zZmkJk#w;}F0FsoBO&qIP4vkH}5 zx7h7{3vy@Vy8!3axchJpf*DgBQ-xBx!DKEAw+}L1##YUs2j}ScX1Y3dpZDV$JUKy= zTbl0qD$`|<&C_%#>|L4k6<5YO+#n$J9H#SDVLIJ{6`qW1w4E^o><2b&r_F~5M4T~! zt+6I%U4(Z5yKij18qqD5MKKJcLR67i9wez?zrs)wtqL$7d*dV*n8`f-5D+hYev$!8 zq8~$PWSsd}2x8Kw0ed(ppQx>m&lHyi{B@ZDYrkY@z*^hhfM4-rJR*2NLj$@@1G>I~ zaWLZebUZPG@d|X5=O~fUL}>VmC$%nw2XSIH1?UJ8OddtYv+sjD2sCs1rM@Bzw6aN; zWj{azB7dU|Yo;4x@tUYdzB5rr418$+vi~NSDY9h$c`L_1^X3cjrmkq5bV;V^+k?`7A%bDW1t1$qWa8! z^nQBdz+?`Z{sNN6fmp(~EZv$gU#v8rm#YIick3mTKTJv8N(E!MxiVLi5@JjB)02$X zBK?=>$R=Ed@lqSc^Klq&9nEK38n0euJnF~NdVAUI{#$*;e{pnQ%xm%k(v*#?p0FN_ z$LJ4oGDNNlO+nle0)-Ww+nP%VkW&b`haP{nq|Q{M$=nuYYVR2mr~1R7OYu! zpl>PcE%+7p1&Zh0gkC0uH_=U4kmN+bt)mgHIqyIQ{f|Z4&jRJEX~7FM$&T(08^tm`CpaZCH9tL)d9!bC=|rGUF&Yc z9JQfaS)ZY$E^QbtvmwaAVMFlj>}*)w7i?G&#dmK|dR7=!Y2b*U3F*V|q=cSTP`O+Z z^xc9faJa=_!Ae8xaDj8)W_s5qXN7cB(jE%U19Q zaSJS1lQy9zUq&rD`})m*3pB`R(SLrD36rBE(_Lc@vT-{*_*cV*Zd`#nW9kbwEC)I5TEAe!(%-@4RG>=HuOR~qFJr+-XH+KSbaLhHDo(r56{@VTKV%I3 z!GM`XUzQBb@STjpVL#r1Oh9|8;J|uC#>4Sib<=T!W5-buG9VD}pPt5)Ax+n~`=C)y z(?yf%VjOp1S?6JzTbM4E-`uRa?oi`v;l78}nCC-5_ufp(f#~xr_pC-xQUgvYsuBIDM#ByiO!_f; zM^+};KQjqhlP)du@LsvwkdG_358I3=eVCfC)SOq=z;coKLK#6|UZ0*VF3tCPmHB#@ zUS+=+_DSsa6&EIl1p7tVFBaMlX(%YDLLRrVjBAk>V8sUQcMV+_Pk#oP>4!x68b8+i zSO6VnRb44Hrh88(Mfv8`gylLwUK%hS899h7HTU{VKYzxG<@^iYUYl8+HUxt!umi17_7J zbO2Qz*s!!j8z$N?P@*$YrzbM4#-IYV2fR$2dV$FRDFt;ChD;Oo(^4kv8$6%{Z+t1z z*D#?h=&|i0V!J@!_tTRsSfv6*6X}RW?q~C1wq84w?H3k|&XyK@y3B&zv8M%--zT%9 z`-%_Kk55<-W7ViiWEij?i!>mHm_=k@3?oX$=ecMPDyVntkIanfz1mVzW%>q{fo_x# z(B8s*Rr|%3AOh8x^zM#NzpVml`Z6_2$=!ZIH&?$HkU=e#Cc}{S?UM}HaGe%*zK8-4 z_KQjO^TQc>OTEqPH}_v^hxgZ2_DfkYMCvZx-h5wiUt(qq;fMKPJ%-60;tK20Vm8dj zNFgf^qasB~lu0W|bqQ_1O7UE#$8^vA2@){<@Iw^XZ#`6fFhk!@y;zig%a2rz>Bckd zJ`gGtsJh-mr>;AtpPumR!^E^5d#obX1dLZk7RI|UjJG^KVLob4u_@Z*vOHGpzN)%>*b>RSHtBx`f}P-Y zVEZ{Jgk`xNbbmaypNral(K8d{Mb^+*J}jgTam>`+KFNX}>W_#laAKs*gl)tRZWN|k z3TqR_g$aGBmzl5)FSMULes9CCcrgLvb&+jCs7q9CK}Z?FhA?4-q0BJ^%&aTr%x((D zTy{;Ps|XiR9mNL&IY{?2Yy@^_>Ew(*G?igN+?p^Tin+#2?~@4rO<#qr8(j^^{syycNdPV`m5QqKgZLq8eQBi#VVx1n;7 z)%+V_KW;gLbQb*=@qkw`kzg02fY?4g$$9~4wZd4^F+oe~#Wt+x#`PDMdF;4o?JdET zbd~jLh{}WJwkN!c_7(R92>=w~(0HaB%;aUjdh}nIXz@)3G~R#^lEl9-kCqgz z{gQZn7F8EA`*B&6z^_?gJoXTF<0V;7hbPo<;W?SC$6O320T7h3$>QUatj8`!W)Rum zm}I?@Wj!S2Wxe&`x#;S>)OwA2Z^#MTK8qdgE8a`@*MPB+3duP-pC}bzdW`Oi+{(%b zFf$VazP)ifnTn`%9)e;oCPzkdOC)lM?+cM!)Os^R5g4!U5rpHY$QzO>=o~E!9nkxY zlCDP`7Tz!m6d#{tJrrSJy`=kE+0W#>tXJ{~{+5}uxM=8w@h&St^n+coU&($SIfSox zFBtqs{msad)FQ3t=!u}gjT$Fl^kwLm$VitF2eN(z=P#-VhkP>H&6w{*9O86^Au8$H zG*X5elVl~rfarjv0X=tLWd0gGz_3|ia?Mjzp;A5)HVf=A{P-jb=0N8q(y?+@#^F8o zUv4A#3+v!v(a>A$ce$tn(cLy=5^2J=PhiLJ73T$Y_=r3Jv)MhwMwyka9Ig&rAdx0y z@-wkGl!NvkdR<&&`XeJp#TGVIjoRB?l!jtBlc5DQCs>f8u^6-n+=GGnYeO;x7RLW1SIv;Q+aaOZyi4@dU;_#yUKnZWI%yc(Om{K#;>?8 zIDk=ovqt+FXC$h25Vkj}s%&#dx`(ewW%Eo3-uP!3wkR52uX~4|rKjU;W4t+xM zy(Ud{c}z&C-##+89c;8?Q&EBjr2SA!u4q`I&9Dq^pJc&m^k0&i0yYbl;lHGj4#yq7 zvf!eVx7cg(mst=CkQU74z=Ws%OJnakif^)D;RYt;tefIG0CnXqZq=otoIk)2t@#GAYI+*Q`I?mEzV-G|@D4(Cn0jZH&{&xZ9l z7ML~YP?5mKpdZ3G#(rqLM*C@&UX79dllPAkizLcNW*h@{1M2RsRB(c#OP?}UuwE`Y zK7GF&gPPy)eaLQCP$Ws|3jR#uj#iER_Q_G5+Ie~~S^3tW;2mF<{cPNSwvHYaoxI(C z`S(@!>(Q?6zkWgQ6ENqUycoY?@PIlyJH}wY#7ttXl;dWS#q;#+*F4Qf>jgATaP9y- zRBXWvUiH4f7Vf}E5ChKPE-FcaxfzPyr!<60LZ+B<)9wol5-4g7`%xf{8#=Snc=I6h zd94cmk}25edn>YD*f!rfcvy7tW_!smml<#PF0F)p>~P-1+f;iGOmc8mHgF~vX<;-m zP>ce$3(R~X1ScCVt%pPKF`!|haM+V0eX196Z)m$=Mh4y|!#4F#!=H(2Z*R0+DZ2Ss zIw~g51@1^yT^C1fxP2I=YxG6rKoJVsMv?10z|yNoASRQofto2Cgc@W_r^s$yEV zR_?vrc=Lsz*&y7W!H*7XoVhd^>ldCS;L2V#d`J59AOq4wNawIEXBe=!It*xI_xbhF zxoF`{-GWb7SujClofeE?A3EeOI4)=(L`8uHjD@qEzB2eO$Xr)WSED2f1F|G5rM7}* z+(%t}sj7c=REiwyhMs9`#not!P^Vwz)n{Y4L^Aa1z-GfCttZDt=V-+`Rg~j*4VFMN-5{GjO z@4igmh4-EJ4E@x!3$hNz)@-}h0p@8zGR==#pCd0kESS6F5P0e$Ps8I=H61J*g9UL0 z!(mN4Kb^8Kp?_N&ZcU_m&Fzy6h#7FC!-4m2Qz#RYyH6 zSjQ2H*p?>T=9XqAoI5ble&u?V3H!>|CFs*Z?z()zgW*K{a4b3c$FW>y6WN~+wtwWM z8iPM*;CMm?6LMra)q!7@h^y*-POW^bBvnQe+`mk8Cm6669ili>gO@8pz9AT}%7?KK z`fo^PBAgOAetMDtM^VujiH;0x^-RB@<~Q!Z^$cfRH1NWJ+8yEOt9MT434i<+{Q6N3hK zP8{hFP5nDTK2sI;_d^91EOKHH+qb?EWit!y(~~R+LVB36G8_%cfFiu13F~MFUdQi? z2Ht4>NV>{^6$u6y&}^^g73&wA7}nnb2W66wsYZCrS{j;a(XhlFC}|pFnix4Lrnhj2 zX6!()6>tZJ;bmB%S88md#{iE&aKE8Qo|vdQFIXueOHdDtD!X|(^#lthj?L#HM+Vx} z7+#W-gTvtVNfyLX!z$wVXo!m8F;OTBYO1NW^&z_G--QX|RVM5avF<@jc^^sCFZeMm zzXy?rK#7Bcxj}mn24u<%`qq$WVqa2%kiBQ5X9~Ab?iHuU80kyO$bbNmKyJU_;(-(* zz?PNX)PH3jqKR6LaBUL|$SOXP-+?SZS=S&s48bQ>M`pl2J;{P-6^8{G*u>GqQO6c6 zX|v!ul329yf?)BoB1Gdxfd$*%k!fFWWP~8 zMa6(yk341X6>`sJT58yvz1mQ8AgtrUWdC7@p(KCsbS)S(v6!JtC#@B8i zWI5XoZ|v=;6byDHEtk?*5OsO=rA6zmfo=FQ%OOiCUWd3h+!ve|-&ZoKrnDTgT&Rtc z0;UFDeQX1S+W|HXj>D%wIr7Bxb>q2No2M!gTL2TsFPV{1r>2XWQ;|?;zhn3&S$K3A z1~GxFz zH01FEUG#klJGn3TF0OFz^*8|V7^fk~y1^D`=Ds*<&iaG81(Pv=NW5HFLRY^bT??zD zI+4|yPi-s5c^J^Q`HgoX@fyUb9O))8A>-A*%ML&klX)IEnas#(`|(Kz1dH2)x*bg% zGF%CM7|^F-z-1C9E&6xOa(cbWfY7m%0rNhMo%|Pk7Y@OTGoZ2IhqZAvbD{iRL{MnF z_#DgcXtzg-xgjjaS1=rnEVhYC3roO@(*v%4Ae${Gbh!x)xwkR$$Jm;i(2v$NL0Nu=s3Pgfe0Y!rljjyprVvG82$mWx7zPy##@?0|oVzZ*Tkz#F3$n$}NblD6 z7Cd!b=-PAWE5NhEXy#U>A>Clu2nwm%*66r+Mh`d)ajJ5--74)zYHJ+A(w1v7U)4yp z@d8Acg0%UYvr_JO$|~+Ip5dR940p*zEjmV3N2nX!*qQ;%vYsiWz@fB{pUP+) zo2fkdks6W*%O$$a=-#vF8=_E;+)iftf<6*VikoAn4`YN*CBy!|v7rB2( zE>dw;>Y^hPwoXa(n58N&1r?@uX;7bTA7rO0M;H16k0W&Vab#EA;3%l;3C*28{-@Y%~yw?)mGEr6J2i8@{|H$AE{^BZmF?SzB7*KppHR6(S~W>`wQO_K{4qvalmhJ@Xeg`Gc!)<( zNES)k45E~K^-|uAH)c*4Z1w@ZZMr98f^CYgxaq;-9Qaa#w0#A6j7kLnP1S*F3`18U zc5q1WVFp1)u|^jnP**q<&ZgJEhT7;konA)>>)C)#dVvTwq`q+){)-=y$zj(24m2|i z*Zqz>x=9lcNQAM&u-?Z+ClqEtQ%gaG77_{Ur2%Uz@JXx;Ld-fAuAzFLL==h(FF}L@ zUD=ZK5Q#G$hnPY);*iqS-_-4PPyOGai!elh%cB@aE46*=z1Hf-Po>t2RZ2~c#$N^N ztDr!fj(6z~NGez+Yo@=CV5W-uN>W-kV&m9zo8SjlOB7u-w$Sa9PF>SK)H?}%^p`Yh zmz1F^%vcUU_unSeO~(XyqgM(R1!DrgPP@?({UGIidNQJM^<+e_F0Ps!>3-G*W{=K4kH7!bSP?C`$x!u7b7>wlGnUqdf2j zIR%xYF7_rpjf!gp@YOau>9H_ZjlLHRZ=$CUlliC;4TCir6m)aw;iWDXGMMD8FaF(X zZx4gs#m8;T3}!>U*0T}x&~EcK>Tb7`;#GFTm>ta){62@d?xKEB0>enRdPe{Tm$9t) z@lj@DNSiel)-93*NwZ3`p%0J*q2YtsATs7Twji@X9bD9Rqsr%0#I1Dl1IHX_HVB8q z5zR0gx>ZS5CQCU?q-|zuEW0J|}w2pB`no?8G155QqnLKX`H%({$we#BG#5m<}V_*tNrO84u`V zfgwBsI@kvF9&9(dNwiyr-Pj9L)C+@=*v1Q~>Bpqohe1mhRZ@zX=d&l%R)vPdq21<< z)4x-7@%1XZ^~D-xH(&Pm)w*#SNVG9)=V&O`bT^=T_0yy5#wi`IunQiE*J?)77wqOM zxph!Of47_O;X6zPFN~IJcmx*#PCjaDL(PTEf&nF=I zZHDj|cVhCxX6>F)lTy$HnskAC5rbLXyyTCk%}~-!`eFv>fa5J$*bn$-ueXQ6{oh7& znc@KDG?5`V-fHvq=-1cMJ-e&y6_FN(y?kx^3+|*nddvws-B*RP8>A|_eHg6|PU(=K z6H-@QH=&QT7h1V0H_f}nI0IeSutkfZpk5$|fwZPPc+O;TVA6>w99_7e1Hhs{z~20Q zA;FdIN=jq03v~5lOg;3suxXxa-VgoyTtc*~43^My9wo`O4*^d4p&!E#O-M9RPU4m- z`^V97WZGXrCnGs+U^CMm>v7dR`l9WF%@W$EnT!}VN8xuZ(JNf&fKgG4xm zmP_sd8@w9r^W*5(jM{A+r9@{~ep4Iz<4(#(8Zv*kT*%x27^kD7jD&}=U5@3(cIvwy4kBLM}!u!#T# zLW7?kx6xRk09uSN2%+ItCRz>ojKrb*&1&dnr7SYK#Ar1nFGYTf#y(oD$)Lz6%2Q0x zO~DZYD`oaIJ5r-kf=>(*v3S~XuaAX>=(#=%AOliWTIDghe;z(%7DH?qMc>HGFRk<{zmU{IUJnV845 zhI_4X380w|ks6Og#?fxejr5Zl_6pQ#j?TXs5*_Go5%$gFu$XDu&*_)bcrwNXkKO)x zC-cioPN7_7F%N2tNM6Iez4F>#F(@kqr6w#!Kc?K`?s4N@iNj!xQChP)=!YXshAk9H zzWr{r%(~-n@t`r{M>UUX0{2d{Tc9?oNqMVrtre=MK!t-un>ZU4T_Uu8AJpGT{OJ(F}^w=fS5XyjNc4K>C~W6GbNHqNS{ z*37wIj9$q1_e}ruZn_6GSomPIgzUdsb!fNBT4`jPD%EH-$bZK+IktnLUk?H|uob#0 zF?m-Ri8G^>Gzz^L?d@S0jU<{$(M(X4UKz9_tL1t7@=HI|L%hmrAq__)msIywi)*Wu zQPDt4WR@HjZ-P%x8^>SOR%^;SwBh!9y8VteZZstKar$lnTjlwPUTK!^kw0!a z?4q(&Z5RVob5uv9)vPgpqEiz6tZ!C(dl*(j1p&;>eS#Q=cCk8r&S)))KO4>cX@q%lMI&eEhjW5D@^!**X zPXc4OcjHkdp>4%BNP>}4ojo_mB@gLrv@>8Op%3Bx1dJTE#E`2`_gk4XvBvPW>E0g3 zrfVh{MmhhXsI;4oo&P-VQhu@YWUp5lE=Sh4F;*Dc-f($sI1(c=g+NQ?qNe%+=l}L` zlU0c0Y(>Lx9OAx>!;dhWSJ|c14~7E|6lnLipbewwjzoUw(8H0)n2+8AH!*w5Wjbo~ zVO3$cHQ|<1V?rd8%P`%C2mQ?MT}O&YD+UNdLbLw{$EKV5CeOcnEdF$r=}H*BNj#9- z&g%wcV~#;ROf(%i^pLaAJ@@f()1t|vcToqXcZ%H1anE(Ly?c}3SxNU)`PD&IVY0&H zFxq>VqGtvJ4QsmUWGb|HwZ^VkTBf`ksoaCS%Qd~apz2ElgheZ1p1J^rC0N%{8W7FlR<+ID!&^p=?S z^Wjg(Fg+JZ!4>^Gq3wWTc0WW`TR6~C1b*%j)p?=Z-JJXOls;BrLZfTRTT zO>r{W_(K~xPUGfxi5eW2+5x`VzSpXlj!j&5iiG?~U!L zXb**|5SjYe0iEtEwkCS;zg|Z(;E3uWAVp;#lL(R-Q7 zYka;6u@)ft!)}9p4_}TaaeO3T)XfG1dmaK8wL82Rh$OAMfO)syYc1}Y zDlGu>J+u*GeqS}p=`5q|aBZTwRzW1HYSg8Kj#Kg)!%;^qxZE(uokk-?O`jhJRR*fY zoQ}+VAG||8j@ag1cCR5nh3Bg@hpJ0|GAr$E3Gm z+9x!r#NC6PiNqak%udkPZKVv6zun{DLqe?-JX?3D_Tp`NxOxoOJdgS-hmbA<40BqB zY)B7n_kf+8|6m)UUGU3{wm0jyd+|NFVvgMSuquCr%YnD`^<5B4E6#pPo4@+P@(>nIXolu zpk!7)KMvkPLzyR3=rVyh1++lNM}PT&oM9rRL^R@6^L{2Kmy@I4GJ4`0X*oc~gnIgQ-!Xs^iVGb}tmsR@#Kn2H}pi7n>* z9b8Y)MvU)4LSE%DQ72#?79|y&EbA%uJa31(F0BxAR<^6(H=u3X$9r{={(D!FHrDPl z8R#N;brW44mH!e+ToMxx8mYuguzYRwk|IA0McX^llp_U$0i>IUf8>w(Z^yvpzy4l{ z1@mfsIYtNGtbG7;#=D4QRP|VQqx%Y%*6q{qeNf5C%-WGrA0(&ve$gUqt6y2Oj}K$i zmD+<4X}DubEi!e8+JVC!3yJ$Ga8fdi{My^YKT;fpgOXW!#6$es59GrlC|@u-7aGbs z`(cp0ZY~Psiqecb2D{N49L?0?VUlnHoe#RKR3u8H=K=md>VPp5a*P0rtt7E9-lT91 zeHZ`vkvpYwF14|6$A%5^t@wQG&bA*&odH`<-pEF^=l$KeZUhQsnM1x3qB-iguWS_o zKQ9=u3m98Gm2A-#1$;vq6G!B{$Bb1>OtO&!KnUXmB@?Bwz28Ou`EelJ&s{jqg~hun zMj|)yf7>d}mQ%V4Vf32|;h1*_mvzeyRi1$`bK8j6k+9HMIj-D77OTLqz=xpOw9Lzo zBd~!|ZH*8^r-uy z;}^j0@SG^~mX!{0_g$|77!7g6*XNygR{Y5&hbjC1S9;{pS;0tRGBn9-1)_^cIHFts zPJRDMB~VYbIKdl%C=Uw(j7fb`#tG5=;{dpN%xDG*E>+;bs$d# z!hP8sAbfJwamwfsP6Rspk%l%P48aZvVHL79q05SDsW>aEGJ*+{NNE2CzR+m_zHHlb z?8=kMrj+Sg{rtG$8?b=0ac5xi6SU3MALF~a=zRQr6~1Ocu0#(7cG<*YCl?*XH+W0w z2+UIO6SXN2_Cy%tG*M(h;N>bK>54T>S*nP+Q-*mwW|4y|Iu)QSM^HaKZ3t(qU}S|z z-6DjKODV=&yFh==z@6|=nhgbbiq@ZjA+)|hYebuzCnu@Ghg8JPw^PqyeRGcjxSe1o1m*i zXycxswyax0NH~A5;0wExDI}f_ASBG>)5x~Ly^=qd9xPbqb0K2^kQwb*>nV=u^Y!XSB z0zd{r{)3m4U3e32>33Mwc(ETMIc!35Jenh=0RS)>HXAi3$;jq8I^1&n{I~(!qnk>X zjidWbTF%F(L-+x{3n1LQ4hNUY0l+7nlT2wl%PjlPygM z`>-qhl;ehptte~avm#$HxF3->!x=%)i7>q~cn~Hhm6M2eA(C2XT6#9pUDNOFCheSRDedlu@N)-brSsKRl4NX%PEu4CMZpanKv zbS3SH_{&)b38<>nQmqDOdI>sTscJ&m3Pcr@{LMU|1Bye?jgr<{BmFK0?a3d2;{+`w zfsnGtFw+(M^td4$&>mH_h?`P9$3sMAL22~TZ}e~#zy>mHBJAA(!Y3^!(2_8)N#Ehm zM|%wfjl6WVaBkMj68|RYim#L*=}Kukl!1XSGIM&>a}zDCXos51awb;bC?WRGh^6Lu$>{`2DoF3(>km@yK@iY9{o5TVv7d= zpEXq`cxfQj!w>=vU>m1E;SCqb#|DBZw9hBbPYq4+DsF#7Cji(gm$3QqQk@8J!hk5N z8#j*|z(uq=0vOpIlJ6fPJnuTWj`j-(hrnzPx|;p2>!h4g+92~w2~K#@=%9o@%jrKs zY)H7L%7tkToxh-87F=Qy!lI36pWv9>(C>pVChWjz?o8qNPmcp)WbdV0hW_hJh`}*; z2=V-q%lS3DUIlUTBjwYBxdVt#ZaHX*Gp0l9UW152(=mZUCxQd2c{9fl(-0s7z7%;7 zfqe{P2$3zEl@vo%MAf0oxy6r98@7hnF>CoE%mcRm_<)$Vom`7;t|QlTG!s|Pm|{@u zXHOXBJsSGNs!j-y`v6-QOk}Vgs>pwni z_;M61vY93K!vo4T@$hu0^Olp>lwZQjRq)o7mFNk!?>VmSj9U(ChSH&h#OGm?dKr4K zC}HA6OsWDws4j*fcvpxoXG-l-mg8~^$&h5(n-QH|lx0k$VxucCXmi~+^yIG$W3+Lr;YL->a7QZiMyJMp|T zYfhJoGY0@SD#JrPmLdC1K}_LECB~b_4c~;Av1;+6(ht%2c6>U_yG>qwZsFHe_@+2= zJU;IAYwC#{BFPqvprW$`qklXcAWTDDS>)M^Q;P$NJRJFD#SQfSRDlXaI4H*xr0&c) zH~6xJS6V5%#|`0<*`iH^TcV1VKUTh|^KO$@m+N&17aKVq@AgVf^+dRka<`b4!#FqO ztpoRKWAy?Mv+KGbHN)H`{A47ISsmsYM#&urTMVLLH0f7oLK@X%!Xc*H$AK_f;1l^H zoiaQft7CZcUw@Dg@YXg$ICnr%RZqxA7%UhWIGBXIm+E@s^cQkoh46A@mGi)e^q8c? z%So`E1uv-dAF_)1m3|!)nI`6p{OM@}*N_&CWb1?+58ypkkf`&1lb7xQhl^^RKy@T| zj#!&(IJw*q&>9&D(PCX7n!*DDxf;Oh(b!(t%I-N=l*J!uPzE8s%sE6;M^9GCqmnNH zhz@@Ns&O|@8@@b%MStour6XJP$N2sLe<<)Z_4B|XO$Yd%TyWm=bm-v#m%}VVg$Kcp8F@YX} zxYhSRJ#O$)F|u;;g0derIA&w0@dt*!fp38@H}Q?B?D0KQJL}FS@b!@Oz+j;%7zCrDhkxRR~*Ueg}hSN37!M=Ii046b;lfVsw9GqF<7~%Zar|JO0*>ulkYX< z?#Hd+Zi{HIOgtqZu!zcJK^nMGUlISsiY_=!0~#zj!;Q>NbXzM=W?vOWL!iwY*v6hQ zaDMx^0bKCon-{>90Ec64zq;T;N$DlHs{k(QyT(vfp5aMCN1E^wyIQwDC!hM)PH{507hxE!%XeEBz)cR5wRdo zxr`|r__i`uaN5&m)#FB!4~|4*8iH%KfzW!S)G8KMq82+k+&$@xi&~|Kk<+-Gm}oE| zM?ZqPi-r4vTDRiw%xU*H;H}ItWuCxIVVoUP|JQk+$xBF2SHX*#LRK8NPw~`qjcuL| z8JSd?x57Fubilj=SXwMOt2NfwtJP@|3{AYqk?RST_p$Q2%jCf`dNZDj4Rc$LeIJNkG~-_B z^eQ0EB#q%Yi&+qoL4X+%`1-6BTo}lK7??7*RX&-kpC31Xd8-bYcaZ!5U^_k}=B*~r zuc=%GaNGEK&`G!F=V|WjnqvbErT{2^7^r6fK3ktnWA0^8B4KC(a`x;P1HwoQi_AN^ zjZM4>UoSujXJVE9!^4Ix2*Zk}o_8Qg;p1(j_6BWpKS0?ZJ!gIT`CE{`-E(%eb(=}$ z?3-xLG4YPIWw_7o!XV@bR2+eO3|gd`$munN>i7k)n;~nN*w+rK#7Ge}RDGz=)!rW9 zLa;+h!wFRVVb^f#Cpp@kHpby^iF6` zmU+-r#VJcwb3Y&^5GweMl45Q1?!Z)MVggKL^2WmuR`JrCkPNO##twLn*Foqa45iQ^ zX%RGn(6X`CM_;5tUduB|grCzr7-Khuv5yn)mB#|1cDeM$!2!b9NO}Yf;wepnc)T-A zp|1S(z1u#nqxRJ+kIk0#nae*TZ5)qM9PHZ zA*M;#ToG)Zb{R)pn*z6BRQZLE%YcLAS0wMi5pXA$1OH;LfpV7ehAR@sNGPMZMsl?d zoX$Aku;NNktE?5~`kIgnMplcl;`jr~DRT46q2c2)ai#c)|t-{Z0wm(i1U1 z)?C!}5C?Q};&{ff(WRx$Q)WoEQ89oyg0gXc{sme z4&3~5DEPPxI1mkhrzs!dc6K*d9%8QaM||0E6Kq;%>hkI`!0=6f1+(?p?Ca1)SYd>qMyk)JsYShJa8Yp!h*ddn>>`0D9<|;>>G`5KfyLozP z-eK|EK1FVROE`R7hFsf_iwBGD?2?Fav_{c0ac%={Fm`zfPO)LlGluXS_K1?m zKut5Ch9|Y&$XkLEF^;Mbrs5ocn`h?c4H3T`Qkqu**Te51;11>vTRXcYa8Xh|fTI>L z!kca{Sc9cLl16T>$TKMpiwV@!Saj$|1LaPZ6b7KPvF4~kdJ5cpxICT0bD<_ z_7%u>c1M_&9EP$0TpqZYaFe3QaOgNm=-BCKWXmB{5oH^z8}x~|T}RZ)JXF|R;dy3m z!Ii=PT?Jh58*sJ*E;@I1L3jZNfg6rU$S*0mY%=hKPGj$Flknw&(|QY3D`(0#v&%Or zXmhMJVhnK{auI$n*f}vI^>MEh=o(&=HwR7__z$X)&_8K*LT~0tCIpmY+7=TTeWU` zRpU;Z{7)F}T$MYQ=%%5>_MqOztneQ_GnujgW65D}IY z%EdWu^V{JUJ}$$pI^h=b5pE~9LoC9rX!;h-?7~e%xO}V>li`vZirXoY6V~oUqXa%T z(OqV=&9U(6Sbel1qPYZaK^6GgUfWf`C3M0NINQ;_M`tHFrbro*i24nI3&S4?RSd#f za0sXZ37E0qBGSQ5W3`~m#w;FPIZSr^3CL-|dF*Hl@dUU9fkb<)uk|wE@&N9jFR=es zXN>Miz$N)3)%yXQ7jQb{%rb!!Zs;h{xGY#BIKs$EA({(;8>n3g96ODN3mp9%xdpr5 zYq<`&3~qEH7mtuTyB(tTDyjt-h8>BLBBloybg}y3%Z*G)U~6#JRt?=_%Y*hI_1G<9 zwob@JYDIb>XH(?Dg0&AHmmwGZK+f$N*86XD5tT$k zX>3nC^h5R06IhZS1G~OLy_B+y>hGXjZp1FI?H43d#BOt4PN17#5eXkxK}V__LT3li zot3DgCQLj=9>9q+kaM~rOtC)e{y!F@B-#2Qzs{a+F1a)saUA{SG_Hs| z2W~-X_v$YMt`a!>eY@tb{#%_~5ka%~Icmv;(dr8IjDeczjPuIMIZ6a`Q-g#mCN_I9 zOX3=56-FbD!}Z`sDtHCkf~yW6SHaf6VM5sK0JeN`J@D@Wp*XM3Jdz|?GFvTsA(ce>DNGUII*p!7Ei}BWZT@eW!?s|R!^btSq3>pc zt$Raz*v@W+AVQKLS0s%_k~|h$9gA%^ai*TbhH3-3*~ztKf&EItPP7|p!NCltSDj~# z-GXX>+04tZiyLx5&jy=^m1B=0mHoV7I>;bUE4$S_a%cB~*Q#^!Mbb!-&s%R7IHg}^nm&4e-I_ZKgiGt*BkLnYlL!k_HMj!`TAT%tF zHHXk_^h39R})UaC?*cYQ|K0aYWTPeIt&{` z%^@5>cXCCrcL7{4m zoI{O2DnJ8rxJk-zhSM||?B~_#j$3&tmm!x2aRp^e+MjPBqCa`XCh$1T02 z%aDr$xzL^d9dc*&X)`T4C%+_YNfMVNTds32vR|oUu9r+XAiN0V=PZ;P1G9y30P#;? z=&N4H;W#X&;gOW)$jz^a^b%jbsnlpKHNyGbr;ejj4sUz=%x-=tKPgSK7$C3*} z%8}6JIdpjmom~Z8-hzG$`2f0;E5cWi=O_6kmNw{gM>sv&@aV>Pt3j2S6@D?Q0Xj$M z92JukMKIdfhU6B&Eu;zT#lQH=fQth--wuE~t5IvZ3vIo;KS{All3zxyb!0bfICOc? z9zdxb8VzbbEE|h6Pdf@BY0ZIawO`Y!d8IOkZhl4h7x&^WgRWKu{oSqy(4AZn^u#zf zL$^c?Ab?KD>6r7wq02hv0HjkYWi-)69%qr&3gqJjoN}PTtf1!}H)608$W;ohr##jimS)PQ z*v%`{9l6)%%hJLHrF3BD$`N*FS47k~=jE38k&{cQ2|F!8uTD;v6}fWKQnfDB`Uf$W zup{kfva#%#1l^qBHpec{so|bqo-fM__uqD47j`Oy{kJ;1BZAf)SUG`R+!(Lum((IC z-EirOjyflbEOSG4Nf@aW25L5PHI6vPUV3q%gp?V#=`KnLVTU&Q=&o!6*4Y){XZ zCDd!(fJ;Zfom~-Lz!kYAexyrMWV)noWD+{*20@MkU87`BMqwfj1#KqxsH{lf5^mRR zP%is9bae^cWf}Db&!5owxX(15)v1M*6QO%h|0?;IW3n6f7 zLL}GX<6s)Xff(gNLsXaKIcmkBaQ54j+zPn)1@Y^-JztX+uI|xc_>{nf1K>{X2Up|J zm(+e7d)4HYB+>qj4kxi2qJ<@r_)wb_(fC-0E|^ z47ubgkt;3iBmT2$wQftt+}eXi1(gSuTN2aYei3wo2qBU)GHAC#oB)IWfzSasb>jpU z$->AA=Xru|UaRh~mFIF9aBTxF9RYX3(5HwBq?w|U+j1pdyH%UDlKru;xu0s1dCXC> zITI}t0SUU8$%P7(gd76bFg)QL=g7_Phw_}SL(Ze#Oys=nkvl6^yCme;=ZZ2VKj92%!$4Q|{daJ>JF}!i>WvMdus{{`<0&Bb?_M zx_P;p$fc{0Gru7htnQIJ8Kh?EzEKa8#Av!AVv|q8wrD6LkU5SCqgK&M9W+!e$x;ye zdk)BQx4O}j!%ELKD3|As@Vr>vVT;f4GT>Nn1>?2tQzd7$YG_Y9Aoh)7)RjvTtF8x< zlniB#ARQ%gAiyWl+)jO@gk$Aa^=zeXqa!D>&{I!gn_my{IXs8UVB;(>b*lHvt)1Nt z;=In0Y!J2i0h=xdKb9MZXsCCD^@e#7j@#z{);VtTd%-`u>u`f2iGT|}?}1x}4qo%*4QfmT zZsbQ|rY}Mf0rIKoeADsP4TSYJR-aQI4q+|Fh{?361s8Cm`r*Y*GfOrvP(S;p-_Pzc z;L-rD`{=s_iLDFNSU?2Mw;&nxQIJ+?zcj75n90h=GajQ57)ri>V zZc|@QL7U$S?&4G`qh2`QtE8icjLtd>DfUq~nF1sWxf zg-4)zjuErs2if&lYPd_c!MGJ{(@Wv$H>UNkf-P;gLhx?aMq=yPX-H<0^|#3hDWi*o z(LQh++Ux=8^xL{Z_+_;05U>ARtDc;3d#!mF|`lB&8N}O6f+ZVP!fC0Z!D3Hp{pN+SBPik zuuX4-4%+k6^Hs2ATx*0a9Klvk78}`D3kBLAM}nL^O2AF9iEC85t%*TdllTbC8UtK* z#)iRHC)99{I=Q=LnntpDF}g$6|92T^X@KTy+k+O)ZUfPB%l+7B(Lv1x)N&dWkv!PM zo76PL719T}x z2`LITcL3YE67ALAP&w5B*hV5m#hEfP<>qv{1)XlJ-@B0uuB6q(6Ix-((KotnFqZi? z54XT=UWM+c)eCVe12>mx2B$viASBIioVfioBwe>LM+Ci7 zvbkctfd`T6O|Zcy-sa&}u;nFemu1O&NIhV)9g}zetxoO)4slc_6dyMBXmnrdR)}(T zy&N}rxGj)#S|n5j>fc4%w1zq+6SdwvYOPKFZkgsQZC-xvsFkP7QoS{0;D({U1KiTd zwZH(r4mARfWf3+xAwlU%5)%kBBjI!?TiaVDH#|VgV?a#Z4sg}prr=7NgEqes%2R&2 zEXi9NN0QK1zb7-WbaExsM0M;Y46=cEXUihD+rt``y@y{2@|S_RjVp8X>^GM2~ri4 z^XBNhrK&85$4o;?Oh_UrRM`rGNjf)>GbJn4{k8eLbL8fQ=Z;%^x-7}tO3p;C<|E|J zu7w~$c6lMm269zih}n=)BR6u`i&r@%V{eB;bp|J;#|q>_Nc zQ#oU_I*lW42<4rCHZMAN(Ck9c(g3a01JKIZwIK1CNcnV7MhY~I<#S{ls5uem>x3g& za)0NJ8|bD$!j_5K$R4@TTBEM7k>xG(Y~{S@+}E0avde&L6nKEwa0J}h)!+zRkRKAq zbSk=a0oXPUNmr00l0Cey3tv~-lc8&t;{n>^O|#CZ~n<&1)L8X zaBhFT)svfHsKJ^TyJEC~n;a3LMsiy5EmK}LvFS)9<;sy>jHbJ++{$RHaqL@ac81!# z+}ts9Pwq0*%0MmWypQVZ$+f_gWFf~fU>m5ZO*#+iQ4m3nU>&$|xUvUll0^a$Rv~Uy zg`2M0;BcK59dGkZIxjUpIVdJwg_=QNk*MuiysVzw2t}$hPPvdU`tXOyHk~-93!(m*ZMv}hUAY>jv$Yd4R`6L2Uq9m%6Xx=qxSOa zt5;JA4zLQ{EV9Bj=CG*NXM)Un^@ z-0D2*Hm@>w(4K!?l;ce#?2X_`ImnEM6RB~WO;|G`>hkxAN%9Y$Qy2UZD#hr7!3w(X z?(N8SpjmFE-rIltLdmCgdtHV}7wF`aE<*>Q9MGxWA1s;rMR?SS(6<>*BY*p_0TiGo4`F~u13V+) zQw5JJOL*qh;NOtmh*!Zgr0$TQ+NCVQx)>~ZFruRzMGYO09R2OXh7Q1~EHq*9BpaM? z26XzBzeXo6(1|HrhE68gxIr*-k4`>C$B2&7tRAOKD*pCipo82h`q6+-KYc--JjmaK zk6Ie%_$=htYWG1d!zZ#VZbyCZ!A_53EgpK~IRX%MG`@Ylfs+9aX9ZAW13$7K$<#n} zL_|DA$5sckC3hJ*@+aAfv_E<*ehnTLH&S<#(pFh}f7k%(i-cNNj?}u6BO*Xi0G;5o zI+}g_eHA_#qvgP-9h}T|dNK!{%v>^1L>Ss7A0IY+?iqsMyTCUAuH(Vk#(N!-n=5=4 zQp>5j_<9*W%De4I@gX8+_XyDJ_F=$Nu)i7(aYs=m6fK0037_EvO!1kY z&Z+#4FPGsH+5jl!z2_57kLM5oka}ACAqh|{+&*jwrF*1I@LkYvg+AZdU_A^0LTP<} zEwqzUd5xF3%cCWNl0AEbqWdC3yxy(AAM_eadVkmu@<6C^kNIwv`Z%_j6F^~ofh{zW zQ+bI`R{@kz;v+z<0Ob>e7UGrV zSFD#Iq#-|Wpd2BTzm8A@dv7Ff>c`IqK0T5#PRo;o5i6*TW7@jM1c2iDB3q~!_X97M zs{n#}mH?`54^TcmrcK8XwE`Ph;nA3T`>;Wj0a0sPcv3l@wh_evB3s{N3%O!fO1vz| z5gDeUd1v2$8}jKnZH&aIWM5-<8pL1u@nHi9dkixw_^xL6jrBV5!D$V-brxt|&wTw| za=a|h0XJRW1E7ezJ{mV!;u`YfobIPmL|`34WpAkL^bB!~b{9u#hbXYDD@ z3z4tSrTdYWp_7svAz$~(LZP0X(X3Y_U0f;QSdOXOK5Y2dJ<|5P#xfA`+Y#-Ek6&M6 z^P1!9Q?UyHvL-*sT-figdU`~A9?^wL!4Z@@)Bw797yt#_^JLILU93Z5+Y#MAuWg-Z zofj8he|2~+OO*vs`VtW>#4Vwnp3hDf7@Z)L@FV4=ZXPy(A^?iqU^M`f;Ok(e;}S~i zBWNM;TZ;>q<;yhD+X3os+#aCQ^V#VF10>eN^~k)=&BKNdV)+abLv(Nz;4}PaUvOO4 zu)e?+I=&_Uz7QXaa)EllxLZN-ygKNZaLsgqK<$Xh<$C)t5Hj5GC2)&@3Amx)KRD*N zf<{$nf>2!O_2&E84{l)adr<#K7Kl|@wnk3fdXtQuu0nq9f(cLYi#Bx zbj;7;vV1FIFJY7S9pLe_U>F)p8X~mAh!7YQZa&}efeTFyV=@Z@pY4P;JQYi8eC8)~ z$WPbd<64pv6zX1kBAyluH++Jmbjy8sl%USr=NmrQTMgI0=Ll$HgM$d(h>u&wsBs~g zYx!5WEFJA~lMe*a{5?ML6dwi|S!#$qP!HfX`S|&U4{mtU$#@C~x3`KRIM>0my1u^_ z!nnTw372J~heQ*&2>0!v@w8-!q#*F2kXpY9!_eY*`+UITu$o-p_A(wnc^u7P*J;Q! zF5y|I-@05cghxH?0Zo|u<#SpwL?@3^U&oG|IQ^`l&nX}*yI2m(QU+Ven1l-5*((8GdYSg{rzDB2rJ4PZ!olBLp`Vt>NNu7 z%Nn3LQ*d{HLcI(iFpd+Tj?@mI(~4maWQmYuS_<|NgpY3?git?_fe`LGPij#}Q;j8* z3s3Cz9X2l+zP#iV%4GVTTS~Jwt0$QS}fC0DJO1OR45CUSxbDB{Fc)m2dQ2@ZD5V2@PAi6?GE*iJ2h$q4{d_|Qk4@-Q&(z`AU`%?Xa01W# zjQ*8FNSDErO-@iM@c^FFGrH;)6K8Zkk1z`!ZXX6f6@Y>12xLfs2yKV7aRIGM&T(EZ z>N14FsD|}3a(@J!)(g>P4}?T+1Plg;_WQ#I5UzQM6+KYI0~CZCK8RQA`VO1d z3SV0G7s3N47Vz{q=>VS76S^?U*hpm)UFCTDu;GJSJ{s?^Vk)3+M>LjG%)BFc3Lb993vo`I9iJsY+1a07zhkDk51Y6SY zV3)z8krwD;l>GuaEfsF?i02fUj1)h9xD6obrb<6z?wp@qIudb$uH zF*5})X6(1wX~A$qNYqSB_^>wr@nJ&7dh+VaV*83LHmI0f$c#IraO)1`*PW#{14`5vgPdW~RsjVq7D%5KKzH?Bz0q zKm|pFs;hg1@^=t|)~})M{`TP`LWQdhIcP7{1|r1#khS$)zmQk$(XKsTg-|L*M|E|3 z!r$c1k80j#0tNl(Mlqcrbj)8xU}#M6-XT)9^vu4r|X&kF-W;p>xz-x$6+J(ZJdL>Vrje zdR8Z%)nY%!a3zI6Zyz>njFY5@OS6lcijW>0 z=f6;9cV%w)>HXuK`|U0(7B|OSPDe8C)o{~V)0MPWXuOd4ZB?HIlMxk~UH3_Y1yHvp z$6C+^0>3eK#$2=|&Fh~R!jbzz%D(?W6l>V`U!;@zCo*~%F}9IVH(mk_%HKm5@AH%qJg}f7i9a30lHoO?!Tr0|9e#)ef=2g=R zQ7vd6Mb+J*47|UdPK&0V`9jEvpxFStLufYKG;#`sZcOZHVD0zVz@W}G&VExvr*-T( zl1ib!N-PN~#QV|i5p-BcBtPOv@<#6W%oh3Ab_;5%0^ZOxs z4zf2u21=|-#^M^<9Iv04)$Di$P)$TfPPpALeOfc)(m4UQoC?kqx}^p>vm6b88MKqC z$ibT@T!ms8=*xp{I+K>UH44ZlOi)8)E=k6!a?pAP*5(Z?BgQ6D6!RbZZndhL@ALvbjxOqaX)Ip1V zFYG`&t?GKo&qU9479z>g5>?7dlS*Uw>}F&pU_h;A~!x~qbtMM~vmFHOnX#Fn(Xf6q~ z#8D;VffmGe>e}4ZSd@RStKb1NlF2&<*5Lhtww7H5XjPz9R&}GufGcH>iqTao){y^* zrASpJ=x36yo@eI_)_LLi zN?6fCj-8A@$hmz+@}%$_ID?C3Lru;_xMnVk@^vuXAR6FM7*ixr+j9utT^Z765ssU# z3o{c5C-0T+6N31yFKoOpj~9Yf{GMC{clzlM`teg-^Ity+7afXMD1mrn>7-ok z(fNWsug*J10krXec_<{dkrZKe-!=YQy?g(9y;Dxkjeo-#l5Km?E4H>iz4`X~8I1Fi z_Jzw6d2Qy)`!cmX!D(7ov;!qtKm&`w9qO&5R7Vxz(-U3~RkScfz&ozK@$x}?7JQka za2TS&!HITv4psais7R02&#HO^RbE$>N)d@V$_X6k5H<+XFdkI&h;D#3MEUSlD7{oT zG65$n5|`C8163n-D*B!WY9?2*EW+7&#ryXwKCZ&eNb|r(_T2%P*GjDc#An1UgSN48 z5CqUvWPTf|F!-2RH_J9)C8vKi0p}7MGDajH?4$U8nbpo&fD-s)KQaE zOLK1hK1NY>LrwDCWv;-K7sR|z__zwTz=#8|`L=VR^I6fn_&f-~dG<5n05)1}RjUo< z1pv40;b3<|i_S2?1>ho2Ha!0cHH_MyD-OYeltM-;MY~y zMYDlj*o(|%JGmP2@IJif)ey$+%vI0{xkwJ9YnH}R}t9srg zEK<*4Tk!lgd|U<_7@q*!QMYfllbazI=@z!eQw{T`geLm;lBwpGAK3mdP^J6^FtNr~ zl>YxC%gxC0Wh$5s;HX&rIM++diz0Nz$5pr`R|anB2)C1)A?Km#NSrKaFS1X%QRo+_ zB|e3)9e8lypm*3oa~aunpCtrd5vk?8fhNJ$66bq#QHw74xC}Hk;~`gc09rV?7jl#+ zYstJ9{8&!8a&_fA7Taih8eU29aNi;%s;trrrkFKxGn{Yv{d99wq9D>!!xnJUH0B-eEvan}-~<;5LFGSNc#x*H@lgR8Llg7EMDPOij=y}TUlha;sKH3|%3 zMpXGE%#2^l^ZaL4Zswq|+EwM^7u-6Ah4vG?DEcB35Us^^BiWAS?!zp&?Hq)+MsN|S zCahTziZM%oO)#bA>@n4V!nl#YT$k*@j{V4u+#1AS5L=_i!kriD?NFiAtJ>;& z`H{BiH4@i{!H)T^t)KxFyKUY5O&UKJ$s{9s5o_PYTyMb7dKUVC_t=^~p{fGXXV<^O z{qNn3PdX{SoMM^%;84^=LZf_l#X?sY(T}F^5m9n}A$)R)NbM3w@g89`J{J+aT4ija z2;-`H#7p=h3TZ<(W4zOGG0t^yK}~98(<9A}O{W|o)!s*0X$~DZ3iBWzOW3Iv4F{!2 zJW(ZdNXUGz4vi*q>zLOs#;&23#t+n3MCJ(_(=2f$eStU zfL8a?|NPiKSzGimYH7pP__%o@b{j`bPE3L5x) z(1D;S5frqP=5-Fj@;&gT5nEN>sNCzw)Jhthmm3i0Nq5P8w)v|CMyNNAXin~t?19Es zY-6*v!D3pqU|85^f19qYayO}`TcxUE?NeYlFz!X{4?&OKBS23!5YunciMo*E$q`A|XFF`}T!fSF9RkK8C$&AA;KTJv05?z`*TR4>obye%usO;R&1hFz^y2JU;(_(rQBaO(+0RZ$zaZK+!jlH$`Qvu|WO z{?P}JW^17FIlaHU!p4wMK*_ExsXYJGD@%VM#8q+E_=MVlRhW!&YSzQmx)Rk3 znFB_#MFS~tvWo1sT& zE0p$aGW#g{zbj1ufA)SQ^?VBXmO~Ua-`lo}Cuu9GUck{B4k3-BI@MmHG(-aD1yk(X z&KesHC*Cr)i&4k7r&%g$n?t$f;X6JVTVM}V^3!U~q=Jf5ImRCI{-XGP->=Fi@&kz{ zH_UYZN@QOtTdsW9r)eAMk_7t?P&4K@M#GrXtDB;Dn3jRIx0JQ8>L@z%gywB$F{Qn3yULHFeM8=^XW$+&2 zdXI`^IhoGad7+MGU zo&UEZA}4+!D~UBIGWOXQCP=tjs^~DonhTheeRSqpO1ez%R}a8aM6N57@h8j?mvM;d zF}m6VEi#F!6ao@uLC?B}{VQ3n2Gf z)!DLFy&30At`qVD8e5Z`+&FZ`ml$6+NRc?M>>1d`-q4U@KqCAE(GpP;Pj;TCXp!Iw zj|!8@RdRJsO3?9u=!*7)Qi{gL#^HMJOeU!gz<1y3`6tA0jmyCiIZXK1h(~5*vjw3w zeOVQ!oUnNPr%;0^O_fT+eN&^AHPU`-by;R1%K2-_>u|;-55VV=oOOO`k8qDhX)5oi zsUh<6Af8<{)E~b&?J=QV<7jT6BuI@7HY_w@wK6nN zJ^mscD9YkfGW?#Y3qI`w^ZeRHfkHymku0q83&d5$CXyCw3@r4f`Owa_rFrch!(NJ) zW-qCt4C5tmbRhvI=y|U`8DCi$5dAMH>4U#T8^arjc6}q7QLRfZpSztJK)}OBAk-q1 zO9_=nDlOvrhy&3QsFHI)5rvE`%V1<8hkN5Kc@d95kX$-@clKw5=x4^R{QdrSkn^x^@UWfdO!ZUsoA5TiIn9uh) zYnF0~U{f8EaP{*iVu*;ayBa>l1GNb)jFKkt^e}x!`;7xHxZS^%v?slGITQx%gUISkv z{nw<gW_Qh zFp+fiDHYWjof8b@GAr%U5y~o85hVl<-r{6-MMD;)Q*6;_KUqSkx7h2-CzTMfKyP^PYJ3`*}W5I#w? zv4-lwk%>FiS_l;Hoya7BSTgNWzc8?ekPNpa>#~;jkzQybKJ~8tVTPUcCqoH4^$;yw zf(24C1@>vkgZWmr6Mx6<4s9$$Cx_8qu8N3wiJub|2mZM&4`cPNL_nmZd65Shl|ZA& z=Gv{p%gw48gnzE@U7^TPgDZ)%|0H`RO#1+J@sE81PC-FOeYY7B5xO&LZhR{VVDD4sIdB6F9jbS@p3ugfDAUGQ zcY75rMIo3?49Q2nnoR$Xu&e+?0E2g9B}b9k<%BozNJTC!IluRp*)5{*RecLW$eUvg zSJXH0h|gUnlsJ@=eYsNAdECLAp0;AiYj@HuvxD-3t++Me049iNPWE|{kIYe~?@iu+ z14@;I10?=SjD7zj!k_I&e>j&2q)qiQ@{bm!uA36a@lCOk;Oqvq%Q2C4Glqk*Z*F5) z<~C8@#AQktDJpF2TG~!R4`12tBP&lBm#C77s&yASXTtm9PjKVTuhT6{I&r$p_3GjS zwP;IrW_(K`KYOo4IiUqIBCF`YO7I8$d*~mJj8?w9&4JFo=A2;wBxC`?Rb@IE1s&~% zXo$8-%s;WNb4sBPG@jvzdB?H#`=a2(jZuV?M5U?pm+cA*YdM0?|sTf-=s zK}6DnsUgj0$l5(};F43?xO|_hf7LW@C|XuEaVnSYymb>ra|6XJAk5VS$Qd1C?}Mm| ziYp)oVs`gB+~SSh7yKZMMYJ!8^)`!a_eN{cd8S$If>|;N&%qYGk`Ah4+HA+;l-%-*JtW1aP80WsZH%=aOojQ&QUSP{pCQlP?7U+` z`8NIiSZ3Pp-ZsZFR6gSRbArakj+<001nA8_vc-a}71hX}J|`G)I`ka>GQ)0XCVkSu z`ADg9_$a7LGWlPwQuq+BuV~h-aSs7pQdfyQ2((-b%WX?pU6&SnD6;%Yo>5#N`R803 z^TDpKK?}8>o)H#}5mY-6WPG`1$sueLegb6%m4h&Hb<$h0n|;um;bG}FNj|p3bZ}fB z&h&#EsZfG9F@Jqyl4LjQj5Vx+A!GE)0SX-Vzb>D9gTQL62C;LB{Xj|*U@@KZKi$8m zZt+T8JIhnO+uOg5V{4YmZF%QLLAe~IZQ)8;#StcTWio+y)JBaQoKg+^zWk%dF&yLr zFq2>|T3M1sMLUbIoTZuOw4iEw^*zqz@qhKcbY12_kCOYD3H`+Z?58dp3WQOWmRKl} zk|tWRg$aoMORs4i-{lpZ^BMl)AwC`+gGs&L(2f$>)g8K8>ij&G@xegQC$|Skcn&%W z4_}mKZ0o{0KLmt_!3Px*lx4DZmUW-h#Mta%0+RWR$4)F+AS|;zg?XL}nQBT&fZMcNA?jA5PfL z$=^Y+4hq@29h$kpO2vGn&Lpzcmgv3Cro$?71f%@96h?nsV`k?CdKs@nE&-&?1mY2D zPDDSYOUFMS);`xsYX)TU5(yBH17LF8ck-$V6DsRdZ1x<7S4lT)xlaKsm%Fcm=i+a0 z`$tKCG*4$edF&R`5Mk#WLgykIW&1(a?vBe;@1Rlz!zCcn+q%Gf6XtdyOyBNjcGZZM zngTWrfDIV7<;041^*L}xi&4C>8>v@nKJR~UjpM9dqf;1$VA7=6!d|KKt0I z3=(#RMmPTBnfKP8Z3fkd2K?+~WLrn%!tX*rC%%oGLvDhBZv8fAvIv4QX~RjQg@sV6M7?jI<1pAsHXt$;fy(Haz$Oh&aE$GyQTjOy*wGl~ zOk07axhEQ!4u+c;vAtHH10v+-KzCAb_RAa3J&%Bq_4z=Re-+kY^6xGo@Yg~9E_Lpk zp%tqZm@$Ef<4Kdr!=Niwy1N(*Zf&Y%#5@$=9lcwjNUeILz&>BZ>M*(*;Pm|?l`A2y z=2dy#X4Z|`6FoW&p9+N_@XaqwtUvaQl+Rw|5nq>gV#1EiJ>H*uLThfMLEj=J7?j{W z?9AQ=>-KbGu_)y(;LCLI)Jy&X%#D#u8P)#^sAqHefVXrD4L-T(6t=YZnxV$^D)-g0 zF*V|-!?%^bdxhR=K0;V*Yq&Dc%5*^)MmHA96Kli15Gr~w#&RlGE@qD*)D(NoYyx8Y> zS~ckbbSGApT1^oiAM|p*{tS}ddSK6p08L9W&6=pX;Md{4TLPA~VSIxL)xOOg zz3n;?3{%D>ZWYtXZTEla{fQqW5X>LjwBlQ{=r$Se$)R`W9}=pEf0%S`Qwza~QKw$_ z$pCo)XY#OQiuw5Ai0*EfDasUrHT_RhSX^4xE{+^O?+CO zIp>IkWs(k@I+BrjF=95H(^%8)J9i`OE)&WGw9^pw5mNN(ee-(v2+Vu*`MePUlul#x z)0ZIUxG@Gh{=kJIosK#J3U(6t{%MH=X6{D$NG4Jc2oNk*apNx4M&0%ZR6xyNROA-o ze$ZW(Sjp0<%rN(eM*`;rObhwNp&s5%i0*RrL!@q6ZyPqDzLf07GtrAA@}d5}D?aM^ z*IJ(Rcq_Toa4sxJkiNcoz`l&vDr=%KE1ifUIqFsZ)cxu5t>Sk%yFXz3<7zaC`_F9| z4cfwG*VS=HcOBJ_77vIscNck?1*o51s`=DrF}dmI+}7#705TQ*gqrGHza(Eps(JK{#^^*K(d|cKcafd+pfOBlvvx~?wMf<*Rp;M*p*Z&( zBsH33u|eSoCsiKjoOV1F?CKA#7fO&U$=>Bzg_?{XjJr6VTpiL~_WrK*)&>hWR zN1m@?#V^K%yZGVOKnm2_ZnQ#JutLSSr5d{e(U29&r3 zHb<5gi@dq^+zZ}UrFEk-!wwy?GfFHzGYIObvys>D^5SyBRTk${>gf8W-`NL{-&E#=IO{k)9G;qS z%2D{RI5cJ(2u3?g10x$^p8k2>)Od9957o>{(_i7!2?LK z@`UH&2`wQcqw1+oLohGt9(+A`%3WK8gx4G1CTo1gePoXLJp|~-MJ(;6%6Sb~ni=>b zPe^FTM%BZ0Ri8L{p3?CYy_I8kHt}5)+94eP@`53mxFqhVpv0w(|2OS z&_?LtxKFY;JwQ;63b09x4coM^@;Mt&QdG-lX}UsWItufZwB`x7#*BVx4z*kP5Z3K{ zZ}lE*RXfy}`vJ!3p(+hWNt zjyTzid_OOMhZ6rfwKaskG=#Ig4zM;)?d42ttHVW-!ool*ZH}_ogngzfGJu}6x$;(T z$~Yn1MCILICaM?`*QPP}MY<46Q*Nj*%|Q)b>r3p@QiO%XFt84^eVok%u>jd9e>^k@ zYidy6xin8flbaHbQ?Ee}Tg1@QI8))%_2rF#AcQd6QQ1N}uZ1q-4k68VPGcNwnR6X* zwEU=KcUTP4&L_}kBKK}!buo0jh8RzryK>qX<2V8lr2gF5?hnbRiuny1>&#&v`&?k`CY-p!Jh{>;yIbwY9J93dnR@2T1ICi z&n(q`%okv=7;}Dd(5hg0cucuxZDwg=poBk~^JtqYcdhKoWx?(lt@Yv14Jmb|xS4L9 z{V+Rf8$9cR&{_WTswy7~Em&J{{{ zFgY!l?90;?O0)7oyd+5&S87s~PXC%;+4@T`ic_6Yw(Qqyc>=c^$yu3RH}w()OPJwm z|E>-aDvR@r#HRsZ82oanw6wO%@YT>Je#+f@a(i%WeaEJ(Uk_X+_pFiT-UD?{!15BP3Ci z+dwEf7o{a)wa5~)?3Q|o(hyDcVW%8^N~~gXcyTV6`_oRPC!>OsohxfqTj?Na75 z3oZBeTAq~hZ~Egfs?Z90%JuDZiH!=4SW^#EQ{#wTOV;E2jTOeSsyP9}Ke_zuA6a;s zNs;ej1H%b&oLqvmmBn_21Ul@QJqKffISrl$btlMN;Yae=q(V3I zQn9A&5CGI0P?3izShzNXL9x#qFx?jQ)OKV1qZy3V;V0ixP)mSih@DP&5CHMex7V_a zz%bRt-Q=gHF{K=z@A?-qGa0S(EShMsN>aw;m8Y-cNetNY7aTd@2Kwe-va`#E`p>4; zxn&!gFu;FD2-T_8P~?}{O16=DSXQb@`U1oATVH%41KCUpGr2gjEYRL+RIg7aVH(&Z zF3LoH(Cdf@OVUl$2Kw`U_gDoC2p^uH24d)_&auE?cbs(<;R;+E2j&sPFZ>M4((d~5 z6g124Ex)eeV@5>`u>n}t)DI>-28AW}Qz_S{Kq%dN|Chx39c;0yI)n}_>Zb81p0g~v z6~4LMtT0GhW%{K%Y)b$O*Zois`|rZ(pe6>pQhoeCZsv&K&^17LS877=Byu|iqae2Npon`aPnre#YL`~QFA@z9oR^0IkUHzq;;ggl>nUtu)7wdh!L`)! zgVqqx+H}lp&&56mXKYExqV7lQ`mxp_-9_yQgBbV=0v8FG*(c7x_l{DIq>#=rr0bd6 z4m+Yc<8FcwC6$*i)B^?@9)FEtuQ891{o{igtKvO82kEs4DNXCmvK_&|vL zN5%3|pj@ohs}~aXMowsdWw@WQlUyqciRG)q_|T8 z$ERTOx!hz6 zgJ_#VGg-Hf#1gR1a#b=A#)3`C)8$OMKz|LFhIEEc zUivbn{k?TJ{@s}K7ZE6eYRn2LnTTozaG`02rH9S_gOY#5zn&kyM5GM^P|TtDR?F*c zY;@Edd*#m}orQ=`)ZM-Y$V4ao*pIp(LqSB?TwP*OZYG&s&9lT0br$}duD7W;nRurKrGM6|P!)+c(||%3 znUh}%0;11%k1?4sq;sF-hT`gStFmk#FKXH{c3?v-wP71P?(q)pjUJ4=qzHegYipz> zniSWO!JpNapyd)_ZPTKSPw8hGAlTJRGHBQ}InGe(ifYKuoB;kI;;sgFmgmL7ezS(e zcuzDN*b1#0mLtINP~xG)LiY`#AJ&lVGyE`WVK+x=(gWo30Oq6T%ZOV+w@LWv$(rRj zM`fjEpWu4ktbf6l%E`Z`YMK>w`ne6p6rMUxpQh`2oqC@c?r-sB)6PhJVn5ZuGp2N) zIxbE#EnfBh#VlE8jNGvUhYT`^$kcY~|7Y zlWh~*qjef*o#>3W?ZtxkCHwybSwFIU11v2;BNGKY@x4n5AN~)`8I>-AIL(s4)TQzI z<&9fE_udiQIvA+r8AZ%{Eg5fC;Pgyzg2@ATIyB6uF0#Tepn1By*~EFjqD${#rX|@9 z!=rCS?yL~3>Ft?~G%4tqJUKk_M^~De&g%Tt zd}PZ`R^Derpze39eGBVs5wwPGkmsN-*ri1Cr$H9|Izhb;MAe%LC)xbgH_szu)xxqw zp{LH0Jt+b7`n70h2)&wO4SpMQDr3QV`+O!>+B1|wE7?J7lwwv}qWc}lja~S_HgUw) z>hPG00Jo&*D4Q=#z5DFqDkYy~yM`6ys|rt-X8l@Hny49^1zKxY)bbPj+_%Ls5!nfV zG)3ER|C(r%Ocz&?A}eIC<@;{{)m_2O{trrkmRl7au)dr$=)<*N?XsQhA{GSkStWHhc_|*k{ERWbCWkN*FV> zh^R~xFD^Fkkec*V_WA_nJmelp*z)0d%v+!jRt8FKPNoj%HtyN`JzlmI>@DVDj=QDm znX6+{O@gSFAIlXcA%^|etbmi73MF8)K+!0`V6dv^3@t{5p#+WU8b`ueZ6Y8Dir?FS zR@|gId%UZcZXJnhuvbc%T#Jm``hS`|A`%K~Ib-@0_yXVRs>x~VmP1`z0W2@PJUKn# zf7K?|d-yPr$+^sMt^!LcQ!SQs`znWRoHuZuc1;usHnx){?h0>apBPk1%`~UJ~CJ>01bbCL$glkw= zerg=@?@I!aqGJDG`mRf;pEFH(TBWegH?JbVs72J0pyh>)%x;M=hnt$&DYZ9A`FVHf zb(lEu`Bw<2Er@?QI=l4$W2O&ZnW|J zEW{bJUOPn2Jkwn*QQzFiD`^ zIi6)C&BmVdG?@p7=SG-Pt4ZE;pyrJ-fGd<-E(32p&~SA$BlDpqpX-VtWkUTVRW}Wc zp4UR(>sT)T0u5&zK+y$G+Q|-6jJ!Dfk?fP(t+}{AH@;(3(tmsZw3@%Np7q^Jiz50-af@8TxRLgHoHWrqUeM{l&{3(u+Bt z;0e~ZlRr%9(r`8So@;RKMbD}LN2-#Y&s-!I=hlp)hQNlT$1)ntU!CjVwP=2|9?v}j z)L={HWm%qr5c=t?2gkvr&IPgnJ3z$0IrE}j**=dibUbt=*6La0_GJ(=Hf3$IfCqN} ziOBuH0;C;DP@=_`BP1|waGOL5btNt$Mm13Zdv%Hrr{D>;7?RJKXa@Rsm6yfVbe$`; z9EV!sZ!1%B>8{0XYP1FQRf>1dtUoY9l)clJ>k*`%(0|A8VrpQmZ9jOf;^$i|+H~gr zb~&!b0OGACh`wmC9f1yusZ%M^es5~Ajc1rD_WKYo`@^XF;I`40mOv2h@FZOIJlE2n z2P*X6VaWNduLQOdJ)>%ANR-TvO1T9II9aXt*4~!F1RjeA(Mbqdy?;l2n}sd}ou^L8 zR^cc(FH2UF-gvk{tB5NWLzzszA94GyFefjT`z8x?6aE6jYPo!U45jVB>85_nz~_#Z z<@%B7>r=w!+<5X0!-cT5L9_qgIl~3xd${qs1%D9p+=<YVMF3BIqGCYSt+4l7X&w z&+>}5C$KP36%j|UXk0oaW6*MSyEzsGmMY&M0%&R#(Za&9@i@RbE{Vka4*DG+_G*07 z(P+PTZ9-2NI>2+Aw)hHBiW7ytm=B*IUkY3eC0bsDuE+bY^;kwius}TeSEnbf3X=62 z6fY5oA^qn0qn$`qssu#3RxUC6c8gPtH~#?$$6o~r6{VivHi%zP+xJYvLc^4*oo_Yw zeEoQ%mLD=l5kmbj2M+VClKa4`TN*33xaolyJDo9tv8MoGuzduyG|!GKNOnr77cQ_&my&2vIQpOXj@@PFo$)icvxHjXL2z4!8{fWm8gEw zHuV+ z!8Y7Nrq?OA^wRt|QWb>h%iDEe$tG!J!T)KX$_EMQ*><@_3NQHAy=eYUC} zY>Kn0{0sk!1Zs4F-_zL#!2cwx_!te%?x*|4t zoU$QA&56#PNd_C{)EBwjetw2bW4blhZ!`WxIFOdZDr-69PbRIIbadT>L3QVpW#M}s zgDLTx)jU+m+r7vZa>Vf@s&lxh6m^>WPpP$F8~;nrEK*7rp4~#&rbsGk4&>vwuEH%` zy^;1r&OuI=a$}2uKH_PHRat845rk6Dql%XWx|=hmWpJ?}*9O?Q_HM!{D`u90bbsW- zE6lg@ZIpq@ADUI)_%KA%inZ>upJ>~cIw20An9j?326htFA3?X)yt{6nGzJnE4=j9Z^GOvhRi-qq~kby zmH2$&6fRq|@kNF(g%f(bJnCp-B=t5C5$ev)Py_Kl5fZ%!95TCFUj(mcWfA@`zkW7M z>6;TQJs`Wex^t+kwC0Mef8~kBE^07cY;b>91WIc)N7+c#`L8GiLnk{sCpnj5yC9%s zDM?>0#^dDxes;d2)7|$5#W`d-DwWlUV4KBmgb$E3a9}>Td*64#gzH!%U6wp|2q#qx z@3-bgD&=g65S;i7)=vrj<}w9ix($>5WRI(=S5=ZWd&&qxIDELEm@$Byq`m+|`gruL zq`oiyKFotKC_$g!@I(y{`X=tl7KXhMMnYiYD^wzm;Wt~vKEx}=L0Ko8^xKN712xo` z#mK1oCk}&BHL}p_U)y`896YXd4UR@PzW~k2t$S4>vG1_4JZ4G{OAhxuPzay9=fq)C zs;P8-1L9TsO51KnpzibSO*YgdN%EsCz`*voJ8kZ&sA94B(*G1FAC20O&`bwFH9S*?c$-?<%C(3eX(}tK`DDvY%kXPF*grWe~-@zPh7QuLPoj%kZZ1u%&cIT2mh7 zX>#s#w=oX+Z9AfyissL58YL8zovvOT%;wq);b)F@-vzHSmtV?3B3U@blS}Yp?zQ@xl#|KPPYj*xFaN>ms1bgwgsRwS@@$~Wa$4B>#PH2bX0Ed(i6 z$W-2^zzw9-(4l!(7-HDl7C^;$Cwl|Fz_l}IeWb&KJ+5*1&OOnh{|0zrZJdszfwsmI z^7ac)R&v|g5XTrqUI9k*(;|D^UvE1ex>)nMyc%*Jj6=a5Ms|XiEgMLygIXjoCK=2k z19N3n)%u3YMZ}O#z7ub;g>p*i&L$=LQsOsPife_q))OK1Ws{r5jMn+%8fdJ=5@Xw| zxU@xy;?d$JV_F7*R+MZ3wo$bL8SFYF)hsi6KL-DJ_J@Cze0K*8^;>_6*w=+ovFVRj zPm3=QowWCaFieyV#5?5%1O>k&M@-pLI`1(~uq%lXyUe1Rxm$j>o}>vlJ=Ja#Pe}_Q zWgMW+Jr4zkInY<$G}fJ7uZmIhDw}>rK)o&|PN}E28u_Sz{zW;Oq27X0HE9-&y3!SB zx07XCV?W5P17daID)bk{YW?w|=qxzp0t`&Z6J~YZxE0HtBx~q>w4rY)G2n&*#aT;~ z%CVoRSJBlQ3U-+pHD6cd#=@7Oo(GWb=vX}Oe8w^NPOP2hB@h9P5m5`olL(sGbjmIq zMCR-Aej$l=InB*Wc524MESQX|q2Ee`bf=?mZcKg(isfX8w}%)@3)bCnA~7AVe;(Pu zh)T&_Lt+h0a6fnA|11i0gzxx6yzl)Er-*ADp$)OhF!+patLfPeWoI2>s$^J$$e|iugl_$+ySi(^)2P4U^(OTwo;t^k2w0oUS?8jEZiT5iqiz84sbCRBYRjhAF#9&Pn zURN`bsQI)boOk|3|A{Egqjk%2`{}jh@^HG0AZvgxPb<%+ft-s^VMv|(Y$3qo9|eQ) zpG_+)xPaMM*()}vf&7!WeBTc>+;UusK5Y^Dxbn1Dkvos#_Y`9O!bo3OAGJqjmObQR zE?*8?PlzG}FZ}k=W2EHH^xVh_#Pl&?e|Cig@1`tREw#&S>vC#w!s!XAH80gYp*fT+ zLr%Ts`eibbB#ztYvHct3!JrW#s)(vi3VY4SN|aYJI%npUM9QZ`j2bOm=*5=T zGTGt~Adh@|M~z`no1`|pn}D&PLdL%pHuI9DxJf+iKyqo8@)!*a@7Ojw%Xew#xVuTL z$(SuV+Q}!4`z%EJ;K?WLC5FdXQj!RP}oL zFqmbU(8?xy79GM>#q$wUYoKm-=es81So`{e3J*U|q)!L2>^Aov@xDck5WgJBADihT zVw-DcHH0@3t?3|Vvc|0UN*3&JY(&^pNNFU%wa1Bc6$U=-4D?U0jgI>4B?89#aQxCv z2y{iy5K=;8;7+xT_Dpf5u$0%AMTYLD7NROaHV;i|W_64FiK{`HmGdT1foRD`bJEWs z8);w9A5!P63I7&|Dog{IcVTEy>>MRLM#;yr?w2E`x^9&b`ha=T(XCS3`Bk+?&f0L- zPSUF_uKC8viKW*REr_8!C<}*+Z~14`$T(wQ^DX?7S_cmt_Wn&&EJxn*;&WJ^Tcx{& zbY$)&I6sf9bC4Y07w{@3+8`u7CY@`dX-gXxO=+?9V%CcywisYM4RV-P(zGCuLjRIo z{SDt6H!!3#AUx428f;Fnz(y5M9i-0r&yiJQ+gKYcpx~_rOD%oX1D(D@Ir@NTbtSnY6Dy zoFj3eoQR6?6p8=!7Ud>@0v@-PXYut^eCXaBA zU5YKB*)y%UE2@O+a6xNQa&yI9(P=pP08HO^rGrT!xFWW8!%J;Zd#w6`Mt7Hps=-E5 z2ipQpL_w+7giYm!b*Y}x{011*$96w6=b$xUPjB_5IFqq6?dY9pE z`>*ny+r_gxQzWJ{(!PDy1)jOm`EzJS)wOE-e^ET_ofu!g*G!;>6ozkvb<<0`nt4kS zBIk)kR%1!mDo|6D5U^FbC)29YRVJ|n(V!Y@-Lv+#6#dC&UihYf&%^e6wHW1%U3bb> z@)HAY_%gn0pW9zXpBIZ_)3THwEfH{bNINh`_d8H?<<{QvMJ$(5Ru{ec3tBHKK+;*^ z18D;N!)ar)b7Hy|SyKYXH@>rR-vn4=>;v_(m?=#!ow)307QURA*5Dx?Z}9d@6jbSO zY4bYEmVS?$E7v^uDpoGMxLo3`%@LyLC@KI?vK9KB60w#8bTWwpu?iKfXsnQd%RwprRd^K zGR((?EKA|lhf#7+Ol1)i>1D;nmWI`J0lqr>|MIhhA;p~u$k>2WHMsS=FWL^ST%uYX zf8UJtm*g3G&*Yh+i!PRf&7XI^UjJ9hf7AnVQ{zUCK#?qN{}&iV0;+7wAE1gK_Yk1EYlnA>)k5dD)(-A2kq0GW)S6w)31$DKpL6E(NinfLjMSy zP-aDu*>#J&Jpj^>o8+LO>u3tUjT=V*zYSRnkFdJap+PgP^LEl_FOFXpT|i_6xX1?r z+dapRQJ!_h5WaGDwNOl&3hz?FUo#@ zwme})k=HQ+T|XEUixqU_A*xiYE^MV%5|wr&KAk1VL>SiAbOTy-tVRrgc;G0i%<-S> zAg2m9ASB1i+1P+iV74;>S^{7#6kxuIh^+@_2v!l-Q93OrYh3m4+%CN*T{fENyBMY) zHSBe4Lm$E2ehHkwDzSs=#9TGyhHQ#6;M?Cz`wpierSCVJMa%{IQ1NfVprIG&9ILg! zkyRDR!7el2t@KXVUWH`3I~Jp3F~}>3fMULU{A^rj^JxEn(jf*Eqkrfyr@oHhb-(r% zcIjtumTb?}&uPvWPo-d7_m97$NX`Zk*#=U%09k7ISo>s3wRvQ8Xfl;q;NT|wuQNO* z1}q$+$GfKZUB*eR-%S#ZPZg99&0VU&gYm9^H7nkmW}r|(BoI7Uzs~?Gf%T=LuX@f< zp$92f2gChD&c?R_lp?kV$tCRfhzGRhrvLX*n`#Q)DMfCKcH6=n$%gu9n`R`#s|Iu6E5PZ=L4(42)kG&P+y(piKD{i(@;r4m zE%cjD*9}$o{LC+bWkH8^R@BiX@%Uh%VHVrz1|doOusf7*375?7A&n`PD5lTZL3gMB zz0pLo39jl&Q!(Ufs>bIn$PkA=@d*RxO zDs4yiZX-8u=7_q2&lGfEKY`I0msJCu8n@BNH|^|VMk>U=bk-Rsvv>y=Ys$UE zyueOYuQGCFp1G*Z5r`G8 z#xgl|c28Kknak5gg3Cu0DGFoHe|*1tPA>4-(0omN1k2=ve@TMGA389qSwR&S0cvQb zvWg|sjGWWToS;zZ`4+NO1fVckp#H9WD;j+{lUIW|_s9M9_k*#AV>``?(lFqE4Yqd~ z^azZT9J|~1pCoky`k%tk5=BXVh7vV|g~Ne5f_kEO5Ob43;eH8l8Gvg)4#sqU#%->;GVlOAr#sBCC zW-UsLrvQtlIcp{cRMRLC^frKL?w_7G=iopSBv^dh% zE74`n*2Ddo<5RyX%Z6n4FM3f=Ss-9YXBH|OfT;U%9ZQ{5Ye!Fb2K2FkA`%DpPw=sA5IRx(2+#N$ukE=(HF?gXw>sv9{2nfngB2avx zTjVfTD}BqQ$-!?`Iu_Ubw|yXL63~0! z#Q>ItB^?`V!;-YF6hpr84Pt9B4ncPykPE{|_>c!NU0UHBQ=~MNt!&A8h?HSBtZ3&# z-Nv6TfR^AJN(LZMJTM%Xn*KYjY~H0bpB08nv_V*Zx&|G!h(+Q{6>G&3tpTzeWTFd4F&V=OXP~{weXnsP982G2y^dT1sdO(n!mnE%uBqCmN})YERP?dc9;*mrPhKD zG%KAO(}%--tsGh!bJFDZl-5bhwVaCU2jb?4IB$?DVoyJ&0>xfJs=22Cys_z-!LXy; zFGPFnry9VJ3T4*^I-%9(sSB{9Bd+>KLa8%)Q~LEG4l0g zrV}1sdt%&jo}kMqKSko@b6+YYT&0=|pM7uvGxP8~m7LT(dz^Tw6Oyc*3RVTZaUx1W zF%#$5j+igR)v3IoGbXktWC~AT%l{gGxLYMmbSdC`DSVNaoBR4ulXy4ZTsZZz~9%&>fv zveik|=52-@NfvQ-8%CHsM?_-1KQ`C9hmTr>%YxsxySB)LR8Zyb zDQD>8KnI@)h1@WlK^S9}56z!>cl2AdoSI3$IxD^8diD6SAy3}vg9~H=tsoDN_V!hO zS@#kR#K(o*9;z&5tQ7YysSM5V+MYCk~WP@}|(7nDUiMX*}#7LVhX)aeHd(lRwQUX?FLnxo~)PiEO1kqPhi)qB^Xv zozHIH6vRv+y|7XJqPsw{^AmnoTtp}|yFRDjTU$*L%1_~=xq&l$cc@mKKpN%)HJ!t8 zHRT@Kg|o%Pq|m?a%gQ8u0w_#Z8l#QTfq&AK;=A6;@i;=;5QH06=vP>Iock-Ltu4?! zLC;SX$W2I-2Zx0+Y_~Z%!sE6k-Y5w^Mok>rb8)cmbTB%Uwm7FYpW)83VrF{;^FvRR z@8h5wOyOA&8T<>iA`)FE_GFvn>jbgM`LIP`@J^XeN&KPh``1|iEw~Vj#)_v}tcv}9 z1MADC<|U^9wAl5!T_+(?XQadC%lYa#41Bi~AzB)JKEkbJhP8jg>Y#5KYwOci1A;A=rec+a5=ZIH%Ns1WHw>%!kdTqYM#K=Bd| z{Xeb)19(w{Q5deQuov|#dj)Z!b+$?HZHG$N8m98E9*Eaj0vNvzFPjP#swAJq>qCcxW}wka%?*WjbFDW4&lcA(LIKH_UPtBjErMq5w z@9S7%VzwB>IO9A;Dtm&sxz0~oXE1{^rXP)!0KLgn*6xt~VzAAsH}Btzx~8$K-T^jr z!86l1jBejyJjf3#3$TUYGuh?He_0n*{ug)bd-7 zCkcdMN@Q`}oL}g)F@BIkJb>;OCOiyHx6v^%%Q_f$LVdpRu7crSev+Kl@c;UqCJ?gzgiqqU{o1~#-`k2VaQ1JGp4^>T_ksFC>Ruv+ zHcQ~uO|vHDpvKD?WD!1PDRA|)0<<}r^x{$WcJ!M>c$p9g9Y?Gogc8kTXZ}_#i ze{1Nb+VLcAo$|Oi_vBZrbuc`RD-^so2b_tn0>iB}B+>0(VdXrS)W49zzL&W==26J4 zD$l0h<$1*gcJmKQ!1kZ$Y9AgI{Zs~J{2za)E&V3Jbd$S`g9KH+o);~#iX71Fu&X#8 zPl{~@cXR8GJ5F&fgTd=en*{U~tcn`~shw70m)7Rm%E0ygV1W%5>IKWuuN`F{*i^aO z4}Ay!n$J5tY02F~^e23eK(2>NJ>+8_byx}PuWjx6-cRpKAaZ|TX>TnyBs%{XS2YXO z(-Uur(_^=oXee4l5!RJ3-W9EQgk-RfL0*#0(Iz~8C3Z$B=j?tn0;+Q2sSBGJBGdm; z=?5(0=Zo)}3^-5(P10X1NZlckQq`2tXUlT#B&uHw+#^82%COn*zgr9NaYYwp;yB8{ zXPuA^>Dn?tzE&^Gj@)eOm_64|tmI0wArriaw5L}`8>nnwx;_0rDTNxjqXHpsxin`i zJVVfJb8g|UT>t1im{kBxtVTlX4Zc*Eo3=(ucip^?qQ*m+gRvw!Q7r*Rh+;bSPLT7l zv$j!anp=iP9;f4)b%`FB^%{1KDN^xzlQWB`_<;*;K@1kJL~i_1*ub9P$gN?S#w z{EZ4;tdjS?CT^N*qqi&fVGqT{2kJ_%8?^w8D|-3O0jZ#*oasfWMvYWTl!Xs#Ts;pHD8U+rA(z{?<@ub#6A} zX{IU|6$E+|Gd6%jM=aIYqe+-6idsWO{As_6TXx7AxRjow68fO+dW=pex+0!Z&3R1h z*g8$}4yR+_YfYjF5=~w-opV5XDt?Tmq!UzhLgAimOre8Zc+Znk)0sK@%JG^Lye|Ry zAKr(}tp@YPGTZa`-ralgLRXg99*pV|2Q!Xq+Lp#6{&!E&hzCOOG8D+dCgA}-0hXq| zbzBoWZ^kBRY>&6KNjaA-jsgkaW7ZUX`o3kd$3?O+N~KqB3ZJ&q2VTxm`5uTejcy7N zFt5=(&t)2uHr^TiF8q_fWI@O2IQP_~lxO8xWWefk)R~#HXG1PjeNkXIYw|5MAmgFV zFg4T;Czod?l)59~L?d$I`a%(T)o0xT=W4HcqIid+gHHU%cI{)&IO-V3YalSljT(qI zy^^E>4CX16@0GvYy@ws!f-E;{RlT1751PctLtoCkKqbxd+!Qa8n1hQY>CUzl~j6)*j71S0G4@q^Sd8i9hQPX(}9>sh}k~+JU!WU zmx_#0x}s4P=J>27G@lzIL>WXLhDZvwED&mP?VyoNjInhNj)ir;a&KJ3&29-Qn1J*? zvEq24-kqVT1&b*6lzUCwAF;=;ex(L|+UHG7Eo{}t|Dbz=l5)ZhM{wbWB_`eXJlkw# zzKIAY*ovE)S8hfH?Jx(qCUN7natYQk(yyBZy2Bx11J-fSLDEsLO9IWXgw&f!k z6pwK3apIUDZ>Nw>72VuXg(CbMOY%8KCiu%y3U4U&M|nOzV#^P*^oq^$EdfSp zwcI9XXU9r+6CpsLwdC6JGVM_rrlsI;=8^2uDPY47Zj-CsP{b z4U}Q-)8hK&eKSJ$o~Euq)evey>`h~q0khr*CXh;G0R`Qe^gOrhic?9TL@ey+QCh}E zAYK8DF+_btI|!&Z5-UxBa3qPEUwj^8tym`M7>R~=+lxw!8Wpl4xm30PJDcaAga`Ib=>|YB5G9olWq9VgOTP1*@VLwcJXikaz3!9I;VUaJma`zv^~+!eOUte!{V+`v;j<@LQ~E?wNk|bue-P-$JyhF z?gW5R=~Qm#9LErIN#mD3mRqA;raVzyoZF9LwJ!oVdXZM8a`q6`y>K(kJ`;;8+e|{7 zK5sCNp^=DkKqh8JN`zo~J$;FPzdM78kAaKp{uOu7sd#Kj+S~P<_K_vT*zvROmp&3| zGjuAV>N4o&yimE#zZ^Nh?5KPWh^*Y(q)H>a#ya4U-DA8l?FVa@Bc>$9{|C_D{uz0~ zdB#VB27Kd~mQXoAU+|bnI_Tu3kiS>y>Hl61Whhne`2X%1hgF?MCbUCnBbOZX2q`5; zFOlK#geUyKZaX3I607gI6WdxWdx;e7e(={j5Fdf!=*`)ocrNIX0+1w{}*tH-E+DkH_|ISLZK8tyUdP{l?e5u&#d&HZ0wivGZ z)7Ymjyxyq9EOHdD)vfbHb{e z3*i9j{muSQ>kAtH?`a5cqr2B)klB3zpI1n;h_-zbrP=-;^Slmq*U3N7s6ff@mxXnO zyR7ZY?d}Yv{habTPwfQm@tVm8&Z&&PceTk?{35aMlPNN?OtTHp&?&a}w@t;jOG695 zbnC(7%_olD2FX=5_(fEYbcm3LhURNQ2E(Oa@(?o1(@G2NJ&lwwgg8wuRB}kqtA8V) zJ@c`lmUZf|4Yy)wp+#7gRG`-Iw<;x1sl|H*3l9$v|9k1(8bPz(mf$QHIT?uamTmuc z9VJxE4I&yf)S~%{Kh57}jJ1Yr*zdmo*B#gri`iGaag2gh1^r@jdIom-^v}yC#YQlX z`)6TF#M9BTccsNBdgp!yFY3D>3+I_rF>tXaNV;KNKqB9;mIe$v?d3KewLo?mH|(_9 zAfWC&k0g{c=AN1`g&#DL==RwtEBUqw9F(I9>L2nCtjl+eiw(qZ%3^Kv;4VyMX`r?+ z_ED9ZwjanX(AZGu_T4oydhiQzHK3($A>Q-qeI_Jx))GU|qIRzo*XYD=Y|(IWY#4vdFK9G4T3~7x;pHClNNY zUIpwurPqe^w^hj3742~XvvBKV*wv_K=Nnu?lZ4nF-04#zkx%VHhWK>uvgw5XDKQJC z`XayHGUWKo<&rJx^2IbI-1pm(GOte+daqxA^bcOn-> z9OtFs)n;|+S85iKE60p&dYF2sRTq((Tpeh}GrgEL?F4Ts>8&-l8?zpma?N>}9T@Z& zwpoun+}ztdU)~n&{p`Ts*>xrBSeZ@@`F_m{Pb%dJgR=WNfg4IQb-{&3&M z7#DHGge0d0Vv98rfWP@`OQg(y>U1HDzb-A`P%GBn>|8k^PnK}+_wH%%l}W4)IZ+@Y z42K^i7~$}JjWjpkrX06(6n$uB)u=^Mc#4;BjJkUYWG9>}pmbifQ!_-A;5RRoGA!q{ zm!vXr#?E_rk_>8t^N}};-nr4VTS3vshs=#lELSmCWZS3BOX5JGl|l<#lW-<6U00=hN=?;8aCKfvYiSc2&MVsScq;vn_g+)>clx~Z zfHl?4q)%+ZmHkCWU&fhr4t`@}7)Qy@+jmgz$1(*&Mk5B!BU&zn0X(ss0PO72hG9JP zd31b?XTDd=n3^);#aMtK!@2jrB6lL$tf)Feb6d)Ih3t0i!-UQK<@r`(3}=2o!aLm) z0>!`h?0h>!d6x_7^Smq=k0atz<`5QM*&gDOknzb`l(9;3YY)2!c81fuS!G12rGRnM z`;{wrEi?DLsEf00mGykY(rR~tfAesW71I$RKLccD#FrRAwiZ=FZACJz-LZncJ7C&b zu@03Eet-RAfRw~uY|Qei4pV%qCbUJ$b<3isYovEqA)hRc{BN{oFs+l`KNA#5PN`xM z*{xrF$QNzS{=EVh*w5c!%%YiyyjAM1dGin?j27z0a6!B-iOwZL%~Cou zqg5_+6M{$)oAS7*?@nPn(Ue@OlxFf)3$+M-6oMU9i1j4$ojh|7_LN644?w}H(yq_l_l8zF&_)N%KNiRSvcM*W z&Rd5x;7$92(ecgTIa{)C%tXp%bVqqK=X6xiaK=!f>9Pd0zec(ICIhkP;+eUn!{_-;Hl#8$XmTWJx6itmPrf-l;$qhve+HO>&MLLn zX)~OS8F^W)$ZQDIA9>pfjB%-^^YiVRh}?XH>GN_HBa$ozgrNGO>0uxN`ug%QX$A_U z%|Dd1-HO~qW;9qOm%<@&W?qCF&GCvK;V4f)yrDB2=o>6m00;B7TQtKwi%rsWy&cXk zg65knS0_!M@vPLBZ0mu5MjGGlwBT+`iW8p7NBnX_M32BQs_`uCpBixq)f>eiJ#1rk zQO&nOvLOTL0Y9uBrpATgj}exCGQ?*kn~$pO$AZlwW`NH~Y)Sc=8E7Z9z>km|Z#B|< zO!%UlMt4|^>_#;9`Hd1hEjJIhJ>QmF^@ixa4K@&l0VT^s>Q)3(RK?C(zxPZ@1K2zk zA~;S>fhfzZR>9!hTa0c zoD$U(Ig`|PZKyvL*u)z&+XCIy_oIf*HCB6D|eI7;c^rY{*dZ zfc4MDtn}da+O(K+)RGBcbT>2+p)JrECvghCE1oq0O$}9b-n=n|`f7B;XW;e5(q9DQ z!EjF!=ede;Bdbz9Tz%JP#Sw1fo+#^eFqg#k&qfNSb)0Qw+X}$0LsbxQOYQRCO?Oyu z5o`FRu=S^MxBLuIJ#?Et*}1^|q6SS=p(${FoTYC$V14>%LaCS&Yrj8GqW*kGwkGB^ z6%fmRs|&Yb%{4QjmsGhP3Etop&u`u!{_ITXHZP7Y%Ubvk11_;;AgEwjp$nA2*csmq zpq6y#?g^AiWKcNgNGxe?lF<_eszkT`S+S=zzmqOdbENjU^HbW1wG3JwohVCYTl03e zRy%Bk5nEB=0!P&p6)i3Dx01UQ)`UauBmPJWtd=IFa1e*!=OIg@8vtGJ-N_0t2$nGX z8n3@=v~X`u>SMgg@gIkBlmE8@C? z(0hQ~G>O1W-cXgD)}gRn`DE?Wj4Y3kjCaNu{?DWUGQ1dJS}RR~AD=c8hVLGvWUmO~ zLx>fT0_8|8%bf%~{zmP^S+pa|^pQ4_CN8a9;oDGa&NV0zigiZIHU`RC@k8t7rTu_f z`&xhUSvV_sRaOljRwH86{dvVPSsuw4=BOrf@z$Id!>|R(AXJfDTSzDFa&2lZapLs} ziIPi@5%%C>4)L6bbw$xjr_0A-H>Ekpcan>!(1AR&O!2JA37&}y(Ax4({?^B&o zKaB8DBbO&+d21JziPqje199K&C-O#ZQHG*9bnXH6Hkh{nYYEhLvFzh<8y1hDfl7n= z0~P7;-td-Zm-JAee8I)x2$PbJ%&v7}cYSqZnP$?Bg>H>V@VSeR2>b0%q|!HES<8aI zJ&b{8m-h3S5}nvSI|&30-vPM{|K`qglEOKe32~o^l1M*S3AZ}o{K09SB#V0kLk_Tw zi%0+P4Eik}v4ZpW+1wcpUa8(pZ3!1!a6SXkLcg{!bY7?+-N`Shll~yEe0}~Q<)IATPyjUZ2kT>wR_EFg?rD~%;kzNOLNoydK|Q^p zvEhF@sn=YB-u@S`>xowa{oPBKm{SY`#Uqo}+ z$Ed!sS5MyOB9kJM++cjZ*ZGs`d!tv&#Lm4CDM(1gErl+(1h~R0>gDdmK2G2N?r>1Z z@EV6F4~poF;z~@UQo<{blwzoylz#&x;*@yBp)Ofj^#kP_z-N+`y=T1*wtqv>4delA zjX^$dl+N(ZujP`;UJF!MWb=4K;rS?p3Twhbz3T;q)UP$dz5%@%_B0=jXu`Vo#&x?CjjVyE= zD|Ttu@nUHah!}Kx9V^Jk$&u74+RZ<*|NusYh|KF{LzOQS~;oyB|~H1WW%^+<`%oQw+Tc) z9i2Nv1EdXWJ+54{-6N%O|J;P!*&Z1{wYNQ-OpOW)eW0hFw@|Fq`zO>wYb_0F&bVL* z2|Cm0bdTNH8}^bu%KFXgM@jksXr?GGTA)P^8nzQ6)WB9nEmPPw{u(!^rWvIOPsWnC zc#)6Peoc(5ADPK3GpACklF`N(bc28W2g#ukk8P!ib+WorNm#sqGl~_l+S0bPXsDKY z-ZuFk?t1%*RC@FQR<4Ty4i+2wAaD}r;l-P4X^eBGnZ!?`SZJYsJ+FIo+Nzh_$K{SP zL;g4O9#`K%7xY@2YU(IiwpcD+VL}1<>N%6<4fIB) zqy{u)S@s+wc4N)(!aN-$&{s5XtK1?@g28?n{!NGr!_SUIflr>Z2M7i_qHnc8m@rLr z8PDVC`eS?Bji=Zo3DA>eWHVTe6be9VSj-z{hI%~*g(SZ1>2Z|*G37 z{_5?!AZ^nG33lw@UMZXH_Q7lsC4usZNCs5w>!5x~S>do!V=2WXb0QZzlFW7oiZU=&)44fdh7v# zAb#Kee?~znGl!cZq<+IBc&YuC`NoU$kz#BQ)) z?oSl?Zjomn`%ZSn*qx~+<2h&R=%%mE?AF%1Qr`@jDuQw7IIRw zj)M~#!B)haMV*C+vh}_l`b5G4W^!d&bVdv909p3vnS9TN5M(#+0qY+W;Dw9mrzG$D ztWwpWa-U?M)o@X!>xAgrMKw~)LwRMC1M-p^+%Tz7+{B`8XHUC%WvrXvnP+p$J&Xj_l6bn~%DTFGV(2zA+$Xt8F!NCo8$dP@z6Pqlik?_U z%Z3h>n=cH}@|d6|0kubm`kmpj5Y&KEB5Srh9lQM(n1i(GN75I=9Kd5yBH0)kjkXj< z))4lSz+$dd#027zYuQdrf`FzIrRYT{cA~7xDpc+^s&%j2?t4O#*E78}PF99PPu=B4Q|OSjXg$Vvv-z_ z8{?F%y@A$7{|Ue%`XEOdMvR~-l&0U0OtN)vs5z@xHPFAZ5UAh=86iC6a8TCyFCYm* zQ)u&#dnhYVGt^#poZjPBAJk-wlYop6SHh%??8+99*yQq(Iv;QTc z2+ytek7#!Comb@EHa!|k(_AJV{xw>HUA-}<>67mmf=|=OPbl6g+`8pTwO7I-EOpHMCaZZZpU$#_v&^}AFyj6O zVM+-_Lua9HS5hhBpJ+3v2}LBkF(n})Ud4}-MQjmEay;o&A!%zPB0EZe7|WTIrxhLc z=B_X8->szyHv#EWB)bu)Pa0876iLVo+bN;SyJInfViJABWgJN0(Xs)apNYo+&o+3n zh-lQ)kK83PEGpp9?qx~JT8M59c=^3Ge=bv7%SYegU{fI|2k@40nDGkRaKfA9wBt_g zf?m|23Kd}dtNYt^p3N=J?cHSCYB{zoD60_Cf2=)ac9lsgI+AQ;>-j&7o$L0X~ow{;O3^LgLT3s}yYp^HGom9ALj3g1i z4D=8`u4;}`KUfBV9e9S*$~gxXk3Lm1 z2uX(PRqTo#695D8_}?U{@5{t3$a!G-ZYFORN33H{VO%rU)d-B1J6HSmKzv^vJ9a$Q z6)9J@ry@IY*u2FbM_z08jm27lU16oZfY<=;`~};u_r!{9Xn0ibqC{)5;kR&zov8*| z5l!*-BL7Mi0VYIzf%)9_m1n*s&crCYwmvw9^DFF4tl%oyew?mdRiqU|wV)XL&LpMw zzaH5J3cT#OQlH;qU%>P0Ql7$!l(=?)7a0)(z`d2RzXLe#d+THk1~h+w{H}YcwCpz!3dHi3`QjLv z{J2sE^`dWC3ny|XLwr5YlxGj&^1%{7V&K!tV~YUHlH6`nO#^oX^e#a=bVKSoJzmYS zx@5#<;8IarK0nu2)fdyV+(fTyj{PILG zjrw1+)o+#-O~+6^sYn*`Zi*c8H#gT+J9tL~;p}AAh@oh(4SP&Fharqh1>c9LEJ29fln4kY? zN7pG_Hk?W}n@Jzy|EAD#!yA`6Z|vdjv+0TEi;5oN11^3jni7z!ufF-z+VL|{>R?P1 zX7;2h^#G7oAiC9)X_y!%=?c;Cs}4rM>90&NE+b!HCQi4GGf8q1F6b$8Re(r%2R(ab z=J*sDl0S@10`qAa_||kcQMe;KndWn^G^p5XA($&izKWED7{;Isv16)!iU1o}yIWsB z;zS|uy}fGR)ZX4?Fpj)-I2aOurD3bsue*a_v?!1)`IOoC35T|@%N60MqlnmXiGtfv zAmz^`Nnqtwn}P6f2v^r~IjtE{Ef6kz$KJA0OJ@6Y0eVT?7Di%?+h5^t1OaxG3vT`{ zq!Z*u{h4tS#TKHvX-iPb)Y-F=1y51BT!GL$w*s|*%PfGs+1}dr@FS)BhY8#x_g8@? zp)0dKkHnkYhx~zzp1bNs4jzZGYT(r^qFG=zBji;>Mg9a&dczLKz=pX2rW;wIx-^-f!h!!jGxZ< zW*Iu`2U$#$9J@6`EB2Ta`L%t0`fCc&n}0WytnQ7k)iV#fjrXL;D8Gwi<}$--Z&o=Y zg=J3N6&QN!RrScaRYq$L5A+%Is(IKfV=ZYYsM!2o zN*Cwc@6@FhTAro!@!LV(7atUmCfxW90NTEhqFzrSX(MdwgKftW-zPWF;SO# zJ`?l${Lxa9VaFz-^jOesoO6nHM}Pn&@p9lpt{+!-Yqq?Xl;Z&mE<^8z_}|NogPhMY zsTpk|Mf;RMG@jwv)%ZnVVM}C$Nf&!8#S$zmTcjiV8 z%IeUPN+%k8#SLP4#e_S`h&NE>n_ULqTo+F=#Ue33qlF(r1Tm0;4IhOIfE&iRK)@u$ zOAZ6Q^iCAtTJ~XWwh3lo>yaTU`7Dv1KY7aHBtp^f@$y)ZKd^w)|AB9`rCb}u%B7a7 z0OUTY)v44cE5E8FVX!@7Qm66&hc8FGA?Q*!!_?>YmtGir@UwVPo5$YnTbv6fAfD+N z-zqfl&raJtJ=XeBa5w z=?L&WGtNLNc=Q!;(;X=)zHPMR=$U-Q>yjxuU5Z19vE0JVjtUmI%3r?d={gSJj=zQK zM#F|lNG?_GP%|gZSA(&)Bv{_ig+2QBUN$AFHIyw_5nSgtV>A|f4+dOUAQ8ode}=! z+^TTlfCKC2FWg`pQ(M1BP1+H(QSN5d_kmF zv}^nWxJ7UA7)AOCg3@n|;L>Mu8(iyQ=&}bj&^bM}`mIP;qLL14(I)mu@@@<7djq#b zYzWx-tSd+8oPWKq=(PO!AUx+d=MikL z3o1fOgg$Prh4WNU&Yaw)_*r&Z=WrVSW|8BxPm*ba!yb!NFKe~KD368}ou0PZfn)(4 zCBblzc;v9B)zDK%giPbaTS5plh4udvS*`e%yS%t@Z;-3i9f(4s@Kpgp9}x^MxVdLv zFaB}nHKOrxfYFdbS{1KF5KXm{1?X^}w9+59TMvfFlUx6R6zeE>9Q%$ZufR}s!h~q{ zCb8~$ChJ%Z`wJ7*ZcIa`g?fG9a+=-+!c|pK(|$;HYo?*!3$I%P^Hsnz^LKhuT7o6D zeo?_zIf!iq4>f6wD2EE7eQ@Lx7E+hj_m~o`5|Ma6)di>CK+`dlEC;h>%U!!0L+>-8 z6)J4(uW%+>l&;xq13#2Jn4*ksdBH}kN<#-X^Gqz_U^4~uj`ed7hu@!?+*vTbk=4IJ zsnCI-MDhypp(pqPJAZ{cNGFQ$I2hxPD@Pw2{y9t8aDJD;NoB9x2`Z_+ZVCyv&1%dT zEQ51U&Xt;rpS1|4MZwXKIRRQ6@HPehj!bf%N1c{Zp>n!mX?n!gH8EI&8nMR((q*~@ z1D2+X)^rYVE2=H#qyIaP?2q=yV8~kg91JwDcY%^WvRxsVnV47z6w$9mY(iSnN%%@} zX+Js*gCHzbASk-@q!lMEw|v`$q72c7=~3Gme(iaq*p4~7Z_DLgyd56*Q(0=^gM z|7$IC+um{gLwzJeGHP8wNhQ>4?>p%=SSFm9@0=JhQ~Tg+L^-m&QbdfD;x|KtYNa9^ zWFQkcbk;E?^M(JbCNTrA0?7}S6$F4z!R)tMdXN1@9cu&WU>NQQY(HI{({%w!hTfm> zioU!NP%52STwj_)|9g3RlE;O|v|?Snki%%NsJ~2DgN)y4Oo{nVmJD(lJP?7(E8#R; zzJ48DDR{^q)B4_1ER4&fnh*k8e8_#a2NiWFa-o)ts_n3^O2m!yWMlsvnwacltW z_o5HP9Gv4l$LF;E7ce}D6VK|6q8%W$aXEVKdzu=O9E^l7E4uE$vG@$@%{bxWl8wg( zkgpD|nvSeeouiw^jyVA;Q3f=saonG_LkQEl$4%!Nn!rw^e>(pqt6v+)J%qkq1#+fN z4j9<;Heq>(D9VK=2GCsK4O^i^;p6*5PEAS>TEeDcA~$~-bKR@wG5ywF@7R2Ky{dk6 zgdY^cGz+~sc}I2rMriwt*I}#ePo0cC^4CTGDvOZiGP|k5dA^au zPx0tQ3G#ccTpd(g*jqZfuKcgLIuslPErm?bi4K)S=sJ5=4ek0bS~IyB!xJ?WzJUbj5L@ z0U{Yv;aFZ`HPu-LYe3`Au+oWz8ZB}nk;ID_cI%SU>fx;V8SWw9CC)LrhgPQP%Z*e28^|B7>nO|lRdvHOs znQ%yET$FV#y(!E$A1^=5c;b_`P5*Z#(m~Pb?Fh;d4;>*%EDlXdj2?T2ML|UL3vREw z9N@4&B~Rahy#Ql{m!pASz#Ap<@RmQgsa`spCgV&Af@{XBa(bf9qT7C>Cy?SF1Eq3|R-K#gN@B z6+mwA!{=E?GJw;DdKw?#7|1?uvu;-W*;X5dwDMhk+14x!6?BACGAltmC7(AvTeb3t z;ZhcKrhKS(iBxz|dP3PQD0YKy-d9k?&Ltjwa6rf6gZ=f8?KUFZO zSuN{C#RGY(+^nVVXz5N^2iJkk8}lCD5s>J-H?jhVQ#rSZ1u!xC0UX#G6jAbS)+3G! zKaKZ6|5IDN@6*)vY>!%nxItpNm=W6E-KzCN0HsTQQ*J=tLL(d=8Q=&EsP!iWVST&} z!h9R{`S|C`l>G(3Jrw0aoV~FLLTOMCI~llvzMj$0r>F=AaexSw&)BLMQPXjbmmivP ziOY&0b6rHJOG5R!hqYmhksB*s_^d!}`e+7>mO7r`3xjE?&^@0r{jkY$-&5UYIvbLu zGRhe{T`_@z_<(9QXHe2SWVx@alGXCOBl!{#pwQWz{KegAC2FcX3@K$hkm{*N)|G8=b)y@o*wo;#4((@%^LMl3G0{%pZrTQ{zUjpwG}|u8^+0f zl@g#f;xryALTxUjnUQR(K6}xzl04*`TT3=e3AqWP`<*0oMUp#^y0sPo#L2Nayu;YU zVjH36$MS{5*p3F52!O|=WsowW6M-~ejwZPcKQ$G0!E7hB<4J{WF-(xx{}(P;M-mGx z>s^%Bgd_-TT0SQx9Dm}s8DELoC;oHSL(L6Pb9Xr9E(#37m-|!3^6ExUm)?T&xH7x4#gF@X zxf3F*#i^P8r3PRR;hL^icrvTnz*;Royj?V0BhRlCOk{sXA>QOkk5ix}%IVC|8Zpk+ zOM8mLHifc?2uKPp>OXlL+vG&oKb@{EqtrF8S!&~Pu=p5dc}3W0arVHH)RHttpAWn( zvpm}>6se}fBZ%f01uN#j@f-!M8hf98f@|i35&3S zvIvz0pDsQ3#kKv;N~0H*&xO}D#sQk#?E221G$hH87ijW>q^3GGlQ+rY*81*wU9?N3 zxoW2t?v%6DgrbO|!FyG`{at<*+E5XZ@v zxWmZ;@k-X8EE3bG#HKD0RJp^H{pG8y`u!V@3~9+kJ^Icu+>vW?pkFUTWt%2x1Td`P zkNy6wYe4VHEx&EGR1Y{$t+8==Iv@CKk}pM$TOzYnsFZqAYxCV`+zx#_IU za~&m&HomuuOe&6ETySA_N0^Kqd^}O`E)ByJT`5`@eDNBZxQImwK^z*Aw^w@s2F&F; zAR2)gxzXK+TW`a&EIRTtux-fvuc-liqd}Dyr}w##*P}9t-Uc#!)NkX?`I-e)Kb8(oiJx^|ZI4Mj{psp<0QKml z5H#9F;056udsJS`1jzN<0#EM6;xIr%J9hS!{N)ZYMpH`8$2KJf7zD_{8J4mR4Co@` zG6S`a6xGp|$;A(KNZ`k}M2OY}w*Q1sWb=dP(6+v8kw`iT!UkWGNGI*q+r+a(o#z?q z0K#T%YP=G9TAXcH!Ce9|6izk_AyM+hK8gO#we37YX+rL$n+!P~AhCLOqSI(v`os1* zZInN;Qkloai^ND@51d_`t`>d0rl^RDB6_=iOR#(Y=@i^f@L)8uzhCjL# ztZTR*b{$5UP`5?45f|a0X9Vss-svWkdpB^nE-id&+p(C!t_L6)9YgJjd52R!z}g$* znNrf}Rl>0}&xOU#K}|hnVDakU0zd-lL=5g@A_}oP37uS zhgmXp%Z;#JADOtZ+#wb2W*J&F1wPFMb{iMUXd7f)56=!5KC)q-hAI%!8DP+5D@AB$ zg!Gje0ou+Bvr}gs>BOM!w{K2sV%7xjTazmPYB)L(udiw0k9MU3p2#V(-E7(vzk?Zo z$k(6X%@`!_nbAkuBd!tr-%yz{`}wxN(SSPx=k1+7`ibOIU|hfstG_8@Ho5{HiMkX9 zuGT@X@cFVo@_T(~I-k9EJVoGzx&lFo`s(#|y@Wv^^wr)L4b6#OsFGr_00000Vq20200Gvq4S Date: Thu, 1 Jan 2015 14:00:34 +0100 Subject: [PATCH 0281/1790] Slightly improved loose object decompression test --- gitdb/test/test_stream.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index eab9a1950..aa434ad3b 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -16,7 +16,9 @@ DecompressMemMapReader, FDCompressedSha1Writer, LooseObjectDB, - Sha1Writer + Sha1Writer, + MemoryDB, + IStream, ) from gitdb.util import hex_to_bin @@ -27,6 +29,7 @@ import tempfile import os +from io import BytesIO class TestStream(TestBase): """Test stream classes""" @@ -144,6 +147,7 @@ def test_compressed_writer(self): def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) + mdb = MemoryDB() for sha in ('888401851f15db0eed60eb1bc29dec5ddcace911', '7bb839852ed5e3a069966281bb08d50012fb309b',): ostream = odb.stream(hex_to_bin(sha)) @@ -151,4 +155,8 @@ def test_decompress_reader_special_case(self): # if there is a bug, we will be missing one byte exactly ! data = ostream.read() assert len(data) == ostream.size + + # Putting it back in should yield nothing new - after all, we have + dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) + assert dump.hexsha == sha # end for each loose object sha to test From 0d22c80e041dbb5d9d985926b39b7bd7a0573a7a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 16:00:55 +0100 Subject: [PATCH 0282/1790] Added integrity test for loose objects to search large datasets for the issue described in https://github.com/gitpython-developers/GitPython/issues/220 See test notes for proper usage, it all depends on a useful dataset with high entropy --- gitdb/test/performance/test_pack.py | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 97c450d6b..d54a74cf7 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -9,6 +9,11 @@ TestBigRepoR ) +from gitdb import ( + MemoryDB, + IStream, +) +from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB from gitdb.utils.compat import xrange @@ -70,6 +75,32 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci + def test_loose_correctness(self): + """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back + into the loose object db (memory). + This should help finding dormant issues like this one https://github.com/gitpython-developers/GitPython/issues/220 + faster + :note: It doesn't seem this test can find the issue unless the given pack contains highly compressed + data files, like archives.""" + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + mdb = MemoryDB() + for c, sha in enumerate(pdb.sha_iter()): + ostream = pdb.stream(sha) + # the issue only showed on larger files which are hardly compressible ... + if ostream.type != str_blob_type: + continue + istream = IStream(ostream.type, ostream.size, ostream.stream) + mdb.store(istream) + assert istream.binsha == sha + # this can fail ... sometimes, so the packs dataset should be huge + assert len(mdb.stream(sha).read()) == ostream.size + + if c and c % 1000 == 0: + print("Verified %i loose object compression/decompression cycles" % c, file=sys.stderr) + mdb._cache.clear() + # end for each sha to copy + @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) From 46a4a79f46ab5a8da97714262095318090674277 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 16:12:15 +0100 Subject: [PATCH 0283/1790] Bumped new version Fixed tiny issue in python 3 --- doc/source/changes.rst | 8 ++++++++ gitdb/__init__.py | 2 +- gitdb/test/test_stream.py | 4 ++-- setup.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f544f76c0..a36fd659c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +***** +0.6.1 +***** + +* Fixed possibly critical error, see https://github.com/gitpython-developers/GitPython/issues/220 + + - However, it only seems to occour on high-entropy data and didn't reoccour after the fix + ***** 0.6.0 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 165993fc9..2a689400e 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 0) +version_info = (0, 6, 1) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index aa434ad3b..44a557d53 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -148,8 +148,8 @@ def test_compressed_writer(self): def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) mdb = MemoryDB() - for sha in ('888401851f15db0eed60eb1bc29dec5ddcace911', - '7bb839852ed5e3a069966281bb08d50012fb309b',): + for sha in (b'888401851f15db0eed60eb1bc29dec5ddcace911', + b'7bb839852ed5e3a069966281bb08d50012fb309b',): ostream = odb.stream(hex_to_bin(sha)) # if there is a bug, we will be missing one byte exactly ! diff --git a/setup.py b/setup.py index dc142c518..c4d9b2ac4 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 0) +version_info = (0, 6, 1) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, From 8b4939630a0d7362e5a6fbca052922d710a87c7e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 18:26:04 +0100 Subject: [PATCH 0284/1790] Improved decompression test to scan the entire git repository, instead of just packs This should make it easier to assert the issue is truly fixed now [skip ci] --- gitdb/test/performance/test_pack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index d54a74cf7..e46031115 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -11,6 +11,7 @@ from gitdb import ( MemoryDB, + GitDB, IStream, ) from gitdb.typ import str_blob_type @@ -83,7 +84,8 @@ def test_loose_correctness(self): faster :note: It doesn't seem this test can find the issue unless the given pack contains highly compressed data files, like archives.""" - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + from gitdb.util import bin_to_hex + pdb = GitDB(os.path.join(self.gitrepopath, 'objects')) mdb = MemoryDB() for c, sha in enumerate(pdb.sha_iter()): ostream = pdb.stream(sha) @@ -92,7 +94,7 @@ def test_loose_correctness(self): continue istream = IStream(ostream.type, ostream.size, ostream.stream) mdb.store(istream) - assert istream.binsha == sha + assert istream.binsha == sha, "Failed on object %s" % bin_to_hex(sha).decode('ascii') # this can fail ... sometimes, so the packs dataset should be huge assert len(mdb.stream(sha).read()) == ostream.size From 84929ed811142e366d6c5916125302c1419acad6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 Jan 2015 11:19:57 +0100 Subject: [PATCH 0285/1790] Applied autopep8 autopep8 -v -j 8 --max-line-length 120 --in-place --recursive --- doc/source/conf.py | 7 ++- smmap/buf.py | 23 ++++---- smmap/exc.py | 2 + smmap/mman.py | 109 ++++++++++++++++++------------------ smmap/test/lib.py | 8 ++- smmap/test/test_buf.py | 20 +++---- smmap/test/test_mman.py | 59 +++++++++---------- smmap/test/test_tutorial.py | 8 +-- smmap/test/test_util.py | 20 +++---- smmap/util.py | 61 ++++++++++---------- 10 files changed, 165 insertions(+), 152 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 90409a138..5aa28e2a6 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -172,8 +173,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'smmap.tex', u'smmap Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'smmap.tex', u'smmap Documentation', + u'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/smmap/buf.py b/smmap/buf.py index 66029cb99..17d2d369b 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,6 +10,7 @@ class SlidingWindowMapBuffer(object): + """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. @@ -20,9 +21,9 @@ class SlidingWindowMapBuffer(object): underneath, it can unfortunately not be used in any non-pure python method which needs a buffer or string""" __slots__ = ( - '_c', # our cursor - '_size', # our supposed size - ) + '_c', # our cursor + '_size', # our supposed size + ) def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Initalize the instance to operate on the given cursor. @@ -57,7 +58,7 @@ def __getitem__(self, i): if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage - return c.buffer()[i-c.ofs_begin()] + return c.buffer()[i - c.ofs_begin()] def __getslice__(self, i, j): c = self._c @@ -72,9 +73,9 @@ def __getslice__(self, i, j): j = self._size + j if (c.ofs_begin() <= i) and (j < c.ofs_end()): b = c.ofs_begin() - return c.buffer()[i-b:j-b] + return c.buffer()[i - b:j - b] else: - l = j-i # total length + l = j - i # total length ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code @@ -86,7 +87,7 @@ def __getslice__(self, i, j): ofs += len(d) l -= len(d) md.append(d) - #END while there are bytes to read + # END while there are bytes to read return bytes().join(md) # END fast or slow path #{ Interface @@ -100,7 +101,7 @@ def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): :return: True if the buffer can be used""" if cursor: self._c = cursor - #END update our cursor + # END update our cursor # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): @@ -112,9 +113,9 @@ def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): # If not, the user is in trouble. if size > self._c.file_size(): size = self._c.file_size() - offset - #END handle size + # END handle size self._size = size - #END set size + # END set size return res # END use our cursor return False @@ -128,7 +129,7 @@ def end_access(self): self._size = 0 if self._c is not None: self._c.unuse_region() - #END unuse region + # END unuse region def cursor(self): """:return: the currently set cursor which provides access to the data""" diff --git a/smmap/exc.py b/smmap/exc.py index 5e90cf722..117664502 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -2,8 +2,10 @@ class MemoryManagerError(Exception): + """Base class for all exceptions thrown by the memory manager""" class RegionCollectionError(MemoryManagerError): + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index 6663687d0..c7a459558 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,12 +1,12 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - string_types, - buffer, - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + string_types, + buffer, +) from weakref import ref import sys @@ -19,6 +19,7 @@ class WindowCursor(object): + """ Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. @@ -29,12 +30,12 @@ class WindowCursor(object): that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( - '_manager', # the manger keeping all file regions - '_rlist', # a regions list with regions for our file + '_manager', # the manger keeping all file regions + '_rlist', # a regions list with regions for our file '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide - ) + ) def __init__(self, manager=None, regions=None): self._manager = manager @@ -65,8 +66,8 @@ def _destroy(self): # this python problem (for now). # The next step is to get rid of the error prone getrefcount alltogether. pass - #END exception handling - #END handle regions + # END exception handling + # END handle regions def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" @@ -121,11 +122,11 @@ def use_region(self, offset=0, size=0, flags=0): # offset too large ? if offset >= fsize: return self - #END handle offset + # END handle offset if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) - #END need region handling + # END need region handling self._region.increment_usage_count() self._ofs = offset - self._region._b @@ -221,13 +222,14 @@ def fd(self): :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") - #END handle type + # END handle type return self._rlist.path_or_fd() #} END interface class StaticWindowMapManager(object): + """Provides a manager which will produce single size cursors that are allowed to always map the whole file. @@ -240,13 +242,13 @@ class StaticWindowMapManager(object): accommodate this fact""" __slots__ = [ - '_fdict', # mapping of path -> StorageHelper (of some kind - '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount of memory we may allocate - '_max_handle_count', # maximum amount of handles to keep open - '_memory_size', # currently allocated memory size - '_handle_count', # amount of currently allocated file handles - ] + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount of memory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] #{ Configuration MapRegionListCls = MapRegionList @@ -279,7 +281,7 @@ def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsiz coeff = 64 if is_64_bit(): coeff = 1024 - #END handle arch + # END handle arch self._window_size = coeff * self._MB_in_bytes # END handle max window size @@ -287,9 +289,9 @@ def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsiz coeff = 1024 if is_64_bit(): coeff = 8192 - #END handle arch + # END handle arch self._max_memory_size = coeff * self._MB_in_bytes - #END handle max memory size + # END handle max memory size #{ Internal Methods @@ -310,23 +312,23 @@ def _collect_lru_region(self, size): for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): + if (region.client_count() - 2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region - #END for each region - #END for each regions list + # END for each region + # END for each regions list if lru_region is None: break - #END handle region not found + # END handle region not found num_found += 1 del(lru_list[lru_list.index(lru_region)]) self._memory_size -= lru_region.size() self._handle_count -= 1 - #END while there is more memory to free + # END while there is more memory to free return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): @@ -336,7 +338,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): :return: The newly created region""" if self._memory_size + size > self._max_memory_size: self._collect_lru_region(size) - #END handle collection + # END handle collection r = None if a: @@ -354,10 +356,10 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise - #END handle existing recursion + # END handle existing recursion self._collect_lru_region(0) return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions + # END handle exceptions self._handle_count += 1 self._memory_size += r.size() @@ -404,7 +406,7 @@ def num_file_handles(self): def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x, y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) def window_size(self): """:return: size of each window when allocating new regions""" @@ -441,7 +443,7 @@ def force_map_handle_removal_win(self, base_path): **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return - #END early bailout + # END early bailout num_closed = 0 for path, rlist in self._fdict.items(): @@ -449,13 +451,14 @@ def force_map_handle_removal_win(self, base_path): for region in rlist: region._mf.close() num_closed += 1 - #END path matches - #END for each path + # END path matches + # END for each path return num_closed #} END special purpose interface class SlidingWindowMapManager(StaticWindowMapManager): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. Once a certain memory limit is reached globally, or if there cannot be more open file handles @@ -482,18 +485,18 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): lo = 0 hi = len(a) while lo < hi: - mid = (lo+hi)//2 + mid = (lo + hi) // 2 ofs = a[mid]._b if ofs <= offset: if a[mid].includes_ofs(offset): r = a[mid] break - #END have region - lo = mid+1 + # END have region + lo = mid + 1 else: hi = mid - #END handle position - #END while bisecting + # END handle position + # END while bisecting if r is None: window_size = self._window_size @@ -506,7 +509,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # Save calls ! if self._memory_size + window_size > self._max_memory_size: self._collect_lru_region(window_size) - #END handle collection + # END handle collection # we assume the list remains sorted by offset insert_pos = 0 @@ -514,7 +517,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): if len_regions == 1: if a[0]._b <= offset: insert_pos = 1 - #END maintain sort + # END maintain sort else: # find insert position insert_pos = len_regions @@ -522,8 +525,8 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): if region._b > offset: insert_pos = i break - #END if insert position is correct - #END for each region + # END if insert position is correct + # END for each region # END obtain insert pos # adjust the actual offset and size values to create the largest @@ -531,13 +534,13 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): if insert_pos == 0: if len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side + # END adjust right side else: if insert_pos != len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows + # END adjust surrounding windows mid.extend_left_to(left, window_size) mid.extend_right_to(right, window_size) @@ -546,13 +549,13 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # it can happen that we align beyond the end of the file if mid.ofs_end() > right.ofs: mid.size = right.ofs - mid.ofs - #END readjust size + # END readjust size # insert new region at the right offset to keep the order try: if self._handle_count >= self._max_handle_count: raise Exception - #END assert own imposed max file handles + # END assert own imposed max file handles r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit @@ -563,10 +566,10 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise - #END handle existing recursion + # END handle existing recursion self._collect_lru_region(0) return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions + # END handle exceptions self._handle_count += 1 self._memory_size += r.size() diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 67aec6333..93cb09a5e 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -9,6 +9,7 @@ #{ Utilities class FileCreator(object): + """A instance which creates a temporary file with a prefix and a given size and provides this info to the user. Once it gets deleted, it will remove the temporary file as well.""" @@ -21,7 +22,7 @@ def __init__(self, size, prefix=''): self._size = size fp = open(self._path, "wb") - fp.seek(size-1) + fp.seek(size - 1) fp.write(b'1') fp.close() @@ -32,7 +33,7 @@ def __del__(self): os.remove(self.path) except OSError: pass - #END exception handling + # END exception handling @property def path(self): @@ -46,6 +47,7 @@ def size(self): class TestBase(TestCase): + """Foundation used by all tests""" #{ Configuration @@ -58,7 +60,7 @@ def setUpAll(cls): # nothing for now pass - #END overrides + # END overrides #{ Interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d07b7f4eb..03377154b 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -3,9 +3,9 @@ from .lib import TestBase, FileCreator from smmap.mman import ( - SlidingWindowMapManager, - StaticWindowMapManager - ) + SlidingWindowMapManager, + StaticWindowMapManager +) from smmap.buf import SlidingWindowMapBuffer from random import randint @@ -57,11 +57,11 @@ def test_basics(self): with open(fc.path, 'rb') as fp: data = fp.read() assert data[offset] == buf[0] - assert data[offset:offset*2] == buf[0:offset] + assert data[offset:offset * 2] == buf[0:offset] # negative indices, partial slices - assert buf[-1] == buf[len(buf)-1] - assert buf[-10:] == buf[len(buf)-10:len(buf)] + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] # end access makes its cursor invalid buf.end_access() @@ -97,7 +97,7 @@ def test_basics(self): buf.begin_access() while num_accesses_left: num_accesses_left -= 1 - if access_mode: # multi + if access_mode: # multi ofs_start = randint(0, fsize) ofs_end = randint(ofs_start, fsize) d = buf[ofs_start:ofs_end] @@ -108,7 +108,7 @@ def test_basics(self): pos = randint(0, fsize) assert buf[pos] == data[pos] num_bytes += 1 - #END handle mode + # END handle mode # END handle num accesses buf.end_access() @@ -116,10 +116,10 @@ def test_basics(self): assert manager.collect() assert manager.num_file_handles() == 0 elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000*1000) + mb = float(1000 * 1000) mode_str = (access_mode and "slice") or "single byte" print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed), + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), file=sys.stderr) # END handle access mode # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index d903af681..b718b065f 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -3,10 +3,10 @@ from .lib import TestBase, FileCreator from smmap.mman import ( - WindowCursor, - SlidingWindowMapManager, - StaticWindowMapManager - ) + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager +) from smmap.util import align_to_mmap from random import randint @@ -29,7 +29,7 @@ def test_cursor(self): cv = man.make_cursor(fc.path) assert not cv.is_valid() # no region mapped yet - assert cv.is_associated()# but it know where to map it from + assert cv.is_associated() # but it know where to map it from assert cv.file_size() == fc.size assert cv.path() == fc.path @@ -60,7 +60,7 @@ def test_memory_manager(self): winsize_cmp_val = 0 if isinstance(man, StaticWindowMapManager): winsize_cmp_val = -1 - #END handle window size + # END handle window size assert man.window_size() > winsize_cmp_val assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 @@ -89,8 +89,8 @@ def test_memory_manager(self): self.assertRaises(ValueError, c.path) else: self.assertRaises(ValueError, c.fd) - #END handle value error - #END for each input + # END handle value error + # END for each input os.close(fd) # END for each manager type @@ -101,7 +101,7 @@ def test_memman_operation(self): data = fp.read() fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 - #small_size = + # small_size = for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): @@ -120,22 +120,23 @@ def test_memman_operation(self): size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us + assert rr().client_count() == 2 # the manager and the cursor and us assert man.num_open_files() == 1 assert man.num_file_handles() == 1 assert man.mapped_memory_size() == rr().size() - #assert c.size() == size # the cursor may overallocate in its static version + # assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded if man.window_size(): - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + # but isn't larger than the max window (aligned) + assert rr().size() == align_to_mmap(man.window_size(), True) else: assert rr().size() == fc.size - #END ignore static managers which dont use windows and are aligned to file boundaries + # END ignore static managers which dont use windows and are aligned to file boundaries - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 @@ -153,16 +154,16 @@ def test_memman_operation(self): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager else: assert c.size() < fc.size - #END ignore static managers which only have one handle per file + # END ignore static managers which only have one handle per file rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # unising a region makes the cursor invalid c.unuse_region() @@ -171,7 +172,7 @@ def test_memman_operation(self): # but doesn't change anything regarding the handle count - we cache it and only # remove mapped regions if we have to assert man.num_file_handles() == 2 - #END ignore this for static managers + # END ignore this for static managers # iterate through the windows, verify data contents # this will trigger map collection after a while @@ -193,21 +194,21 @@ def test_memman_operation(self): # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() - #END statics will overshoot, which is fine + # END statics will overshoot, which is fine assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, (size or c.size())).is_valid() csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] + assert c.buffer()[:] == data[base_offset:base_offset + csize] memory_read += csize assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed), + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), file=sys.stderr) # an offset as large as the size doesn't work ! @@ -217,6 +218,6 @@ def test_memman_operation(self): assert man.num_file_handles() assert man.collect() assert man.num_file_handles() == 0 - #END for each item + # END for each item # END for each manager type os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index 5c931de73..f7a2128a9 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -20,7 +20,7 @@ def test_example(self): # Cursors ########## import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + fc = smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") # obtain a cursor to access some file. c = mman.make_cursor(fc.path) @@ -40,7 +40,7 @@ def test_example(self): assert c.size() c.buffer()[0] # first byte c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size()-1] # last byte + c.buffer()[c.size() - 1] # last byte # its recommended not to create big slices when feeding the buffer # into consumers (e.g. struct or zlib). @@ -72,8 +72,8 @@ def test_example(self): assert buf.cursor().is_valid() buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:]# access the last ten bytes + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes # If you want to keep the instance between different accesses, use the # dedicated methods diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 745fedf2e..0bbf91b65 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,13 +1,13 @@ from .lib import TestBase, FileCreator from smmap.util import ( - MapWindow, - MapRegion, - MapRegionList, - ALLOCATIONGRANULARITY, - is_64_bit, - align_to_mmap - ) + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap +) import os import sys @@ -74,7 +74,7 @@ def test_region(self): assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. @@ -83,7 +83,7 @@ def test_region(self): # argument of mmap. if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - #END handle platforms + # END handle platforms # auto-refcount assert rfull.client_count() == 1 @@ -111,7 +111,7 @@ def test_region_list(self): assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size - #END handle input + # END handle input os.close(fd) def test_util(self): diff --git a/smmap/util.py b/smmap/util.py index 394e6b197..e079a527a 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -10,10 +10,10 @@ # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly # useful for aligning the offset. The offset argument doesn't exist there though from mmap import PAGESIZE as ALLOCATIONGRANULARITY -#END handle pythons missing quality assurance +# END handle pythons missing quality assurance __all__ = ["align_to_mmap", "is_64_bit", "buffer", - "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -25,7 +25,7 @@ def buffer(obj, offset, size): # return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! - return obj[offset:offset+size] + return obj[offset:offset + size] def string_types(): @@ -45,7 +45,7 @@ def align_to_mmap(num, round_up): res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY if round_up and (res != num): res += ALLOCATIONGRANULARITY - #END handle size + # END handle size return res @@ -59,11 +59,12 @@ def is_64_bit(): #{ Utility Classes class MapWindow(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes - ) + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) def __init__(self, offset, size): self.ofs = offset @@ -104,21 +105,22 @@ def extend_right_to(self, window, max_size): class MapRegion(object): + """Defines a mapped region of memory, aligned to pagesizes **Note:** deallocates used region automatically on destruction""" __slots__ = [ - '_b', # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_size', # cached size of our memory map - '__weakref__' - ] + '_b', # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot + # END handle additional slot #{ Configuration # Used for testing only. If True, all data will be loaded into memory at once. @@ -142,7 +144,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0): fd = path_or_fd else: fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) - #END handle fd + # END handle fd try: kwargs = dict(access=ACCESS_READ, offset=ofs) @@ -162,18 +164,18 @@ def __init__(self, path_or_fd, ofs, size, flags=0): self._mf = self._read_into_memory(fd, ofs, actual_size) else: self._mf = mmap(fd, actual_size, **kwargs) - #END handle memory mode + # END handle memory mode self._size = len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, self._size) - #END handle buffer wrapping + # END handle buffer wrapping finally: if isinstance(path_or_fd, string_types()): os.close(fd) - #END only close it if we opened it - #END close file handle + # END only close it if we opened it + # END close file handle def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ @@ -181,11 +183,11 @@ def _read_into_memory(self, fd, offset, size): mf = '' bytes_todo = size while bytes_todo: - chunk = 1024*1024 + chunk = 1024 * 1024 d = os.read(fd, chunk) bytes_todo -= len(d) mf += d - #END loop copy items + # END loop copy items return mf def __repr__(self): @@ -221,7 +223,7 @@ def client_count(self): """:return: number of clients currently using this region""" from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 + return getrefcount(self) - 3 def usage_count(self): """:return: amount of usages so far""" @@ -245,17 +247,18 @@ def buffer(self): def includes_ofs(self, ofs): return self._b <= ofs < self._size - #END handle compat layer + # END handle compat layer #} END interface class MapRegionList(list): + """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( - '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map - ) + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) def __new__(cls, path): return super(MapRegionList, cls).__new__(cls) @@ -267,7 +270,7 @@ def __init__(self, path_or_fd): def client_count(self): """:return: amount of clients which hold a reference to this instance""" from sys import getrefcount - return getrefcount(self)-3 + return getrefcount(self) - 3 def path_or_fd(self): """:return: path or file descriptor we are attached to""" @@ -280,8 +283,8 @@ def file_size(self): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size - #END handle path type - #END update file size + # END handle path type + # END update file size return self._file_size #} END utility classes From ff7615321ee31d981a171f7677a56a971c554059 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 Jan 2015 11:21:36 +0100 Subject: [PATCH 0286/1790] Applied autopep8 autopep8 -v -j 8 --max-line-length 120 --in-place --recursive --- doc/source/conf.py | 7 +- gitdb/__init__.py | 6 +- gitdb/base.py | 20 +- gitdb/db/base.py | 12 +- gitdb/db/git.py | 5 +- gitdb/db/loose.py | 8 +- gitdb/db/mem.py | 3 +- gitdb/db/pack.py | 6 +- gitdb/db/ref.py | 2 + gitdb/exc.py | 14 ++ gitdb/ext/smmap | 2 +- gitdb/fun.py | 104 ++++++----- gitdb/pack.py | 111 ++++++----- gitdb/stream.py | 52 +++--- gitdb/test/db/lib.py | 4 +- gitdb/test/db/test_git.py | 8 +- gitdb/test/db/test_loose.py | 3 +- gitdb/test/db/test_mem.py | 1 + gitdb/test/db/test_pack.py | 6 +- gitdb/test/db/test_ref.py | 5 +- gitdb/test/lib.py | 32 ++-- gitdb/test/performance/__init__.py | 1 - gitdb/test/performance/lib.py | 20 +- gitdb/test/performance/test_pack.py | 34 ++-- gitdb/test/performance/test_pack_streaming.py | 33 ++-- gitdb/test/performance/test_stream.py | 49 ++--- gitdb/test/test_base.py | 15 +- gitdb/test/test_example.py | 3 +- gitdb/test/test_pack.py | 34 ++-- gitdb/test/test_stream.py | 12 +- gitdb/test/test_util.py | 2 +- gitdb/typ.py | 6 +- gitdb/util.py | 34 ++-- gitdb/utils/compat.py | 2 +- gitdb/utils/encoding.py | 2 + setup.py | 174 +++++++++--------- 36 files changed, 460 insertions(+), 372 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 723a34503..68d9a3fcd 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -171,8 +172,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'GitDB.tex', u'GitDB Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'GitDB.tex', u'GitDB Documentation', + u'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2a689400e..791a2ef2f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -8,6 +8,8 @@ import os #{ Initialization + + def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('smmap',): @@ -17,8 +19,8 @@ def _init_externals(): __import__(module) except ImportError: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) - #END verify import - #END handel imports + # END verify import + # END handel imports #} END initialization diff --git a/gitdb/base.py b/gitdb/base.py index a33fb67b9..5760b8af0 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -11,12 +11,14 @@ ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream') #{ ODB Bases + class OInfo(tuple): + """Carries information about an object in an ODB, provding information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. @@ -62,6 +64,7 @@ def size(self): class OPackInfo(tuple): + """As OInfo, but provides a type_id property to retrieve the numerical type id, and does not include a sha. @@ -71,7 +74,7 @@ class OPackInfo(tuple): __slots__ = tuple() def __new__(cls, packoffset, type, size): - return tuple.__new__(cls, (packoffset,type, size)) + return tuple.__new__(cls, (packoffset, type, size)) def __init__(self, *args): tuple.__init__(self) @@ -98,6 +101,7 @@ def size(self): class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, Either the 20 byte sha which points to some object in the database, or the negative offset from the pack_offset, so that pack_offset - delta_info yields @@ -115,6 +119,7 @@ def delta_info(self): class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional information about the stream. Generally, ODB streams are read-only as objects are immutable""" @@ -124,7 +129,6 @@ def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - def __init__(self, *args, **kwargs): tuple.__init__(self) @@ -141,6 +145,7 @@ def stream(self): class ODeltaStream(OStream): + """Uses size info of its stream, delaying reads""" def __new__(cls, sha, type, size, stream, *args, **kwargs): @@ -157,6 +162,7 @@ def size(self): class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object is provided""" __slots__ = tuple() @@ -176,13 +182,13 @@ def stream(self): class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() def __new__(cls, packoffset, type, size, delta_info, stream): return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface def read(self, size=-1): return self[4].read(size) @@ -194,6 +200,7 @@ def stream(self): class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow the ODB to record information about the operations outcome right in this instance. @@ -246,7 +253,6 @@ def _binsha(self): binsha = property(_binsha, _set_binsha) - def _type(self): return self[1] @@ -275,6 +281,7 @@ def _set_stream(self, stream): class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in the queried database. The exception attribute provides more information about the cause of the issue""" @@ -301,6 +308,7 @@ def error(self): class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" __slots__ = tuple() diff --git a/gitdb/db/base.py b/gitdb/db/base.py index a670eea63..2615b13cc 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -19,11 +19,11 @@ from functools import reduce - __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') class ObjectDBR(object): + """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" @@ -61,6 +61,7 @@ def sha_iter(self): class ObjectDBW(object): + """Defines an interface to create objects in the database""" def __init__(self, *args, **kwargs): @@ -100,6 +101,7 @@ def store(self, istream): class FileDBBase(object): + """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" @@ -113,7 +115,6 @@ def __init__(self, root_path): super(FileDBBase, self).__init__() self._root_path = root_path - #{ Interface def root_path(self): """:return: path at which this db operates""" @@ -128,6 +129,7 @@ def db_path(self, rela_path): class CachingDB(object): + """A database which uses caches to speed-up access""" #{ Interface @@ -143,8 +145,6 @@ def update_cache(self, force=False): # END interface - - def _databases_recursive(database, output): """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" @@ -159,10 +159,12 @@ def _databases_recursive(database, output): class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + """A database which delegates calls to sub-databases. Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" + def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() @@ -207,7 +209,7 @@ def stream(self, sha): def size(self): """:return: total size of all contained databases""" - return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) + return reduce(lambda x, y: x + y, (db.size() for db in self._dbs), 0) def sha_iter(self): return chain(*(db.sha_iter() for db in self._dbs)) diff --git a/gitdb/db/git.py b/gitdb/db/git.py index d22e3f1b9..a4f6f54c3 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -20,6 +20,7 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): + """A git-style object database, which contains all objects in the 'objects' subdirectory""" # Configuration @@ -41,8 +42,8 @@ def _set_cache_(self, attr): self._dbs = list() loose_db = None for subpath, dbcls in ((self.packs_dir, self.PackDBCls), - (self.loose_dir, self.LooseDBCls), - (self.alternates_dir, self.ReferenceDBCls)): + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) if os.path.exists(path): self._dbs.append(dbcls(path)) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 374302611..e924080ef 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -57,10 +57,11 @@ import os -__all__ = ( 'LooseObjectDB', ) +__all__ = ('LooseObjectDB', ) class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" # CONFIGURATION @@ -73,7 +74,6 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): if os.name == 'nt': new_objects_mode = int("644", 8) - def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) self._hexsha_to_file = dict() @@ -164,7 +164,7 @@ def info(self, sha): def stream(self, sha): m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True) return OStream(sha, type, size, stream) def has_object(self, sha): @@ -199,7 +199,7 @@ def store(self, istream): else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) + chunk_size=self.stream_chunk_size) # END handle direct stream copies finally: if tmp_path: diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1aa0d511f..595dbf4fc 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -28,7 +28,9 @@ __all__ = ("MemoryDB", ) + class MemoryDB(ObjectDBR, ObjectDBW): + """A memory database stores everything to memory, providing fast IO and object retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already @@ -85,7 +87,6 @@ def sha_iter(self): except AttributeError: return self._cache.keys() - #{ Interface def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eaf431a22..6b03d8383 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -31,6 +31,7 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): + """A database operating on a set of object packs""" # sort the priority list every N queries @@ -113,7 +114,7 @@ def sha_iter(self): def size(self): sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes, 0) + return reduce(lambda x, y: x + y, sizes, 0) #} END object db read @@ -127,7 +128,6 @@ def store(self, istream): #} END object db write - #{ Interface def update_cache(self, force=False): @@ -177,7 +177,7 @@ def update_cache(self, force=False): def entities(self): """:return: list of pack entities operated upon by this database""" - return [ item[1] for item in self._entities ] + return [item[1] for item in self._entities] def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index d98912643..83a9f611d 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -8,7 +8,9 @@ __all__ = ('ReferenceDB', ) + class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" # Configuration diff --git a/gitdb/exc.py b/gitdb/exc.py index 73f84d299..d58442f2b 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -5,28 +5,42 @@ """Module with common exceptions""" from gitdb.util import to_hex_sha + class ODBError(Exception): + """All errors thrown by the object database""" + class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + class BadObject(ODBError): + """The object with the given SHA does not exist. Instantiate with the failed sha""" def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) + class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" + class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object in the database""" + class BadObjectType(ODBError): + """The object had an unsupported type""" + class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index eb40b44ce..84929ed81 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit eb40b44ce4a6e646aabf7b7091d876738336c42f +Subproject commit 84929ed811142e366d6c5916125302c1419acad6 diff --git a/gitdb/fun.py b/gitdb/fun.py index b7662b495..17da4e5da 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -31,15 +31,15 @@ REF_DELTA = 7 delta_types = (OFS_DELTA, REF_DELTA) -type_id_to_type_map = { - 0 : b'', # EXT 1 - 1 : str_commit_type, - 2 : str_tree_type, - 3 : str_blob_type, - 4 : str_tag_type, - 5 : b'', # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA +type_id_to_type_map = { + 0: b'', # EXT 1 + 1: str_commit_type, + 2: str_tree_type, + 3: str_blob_type, + 4: str_tag_type, + 5: b'', # EXT 2 + OFS_DELTA: "OFS_DELTA", # OFFSET DELTA + REF_DELTA: "REF_DELTA" # REFERENCE DELTA } type_to_type_id_map = { @@ -55,8 +55,8 @@ chunk_size = 1000 * mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures @@ -72,6 +72,7 @@ def _set_delta_rbound(d, size): # MUST NOT DO THIS HERE return d + def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static @@ -89,9 +90,11 @@ def _move_delta_lbound(d, bytes): return d + def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) + def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer :param bbuf: buffer providing source bytes for copy operations @@ -112,15 +115,16 @@ def delta_chunk_apply(dc, bbuf, write): class DeltaChunk(object): + """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" __slots__ = ( - 'to', # start offset in the target buffer in bytes + 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - ) + ) def __init__(self, to, ts, so, data): self.to = to @@ -142,6 +146,7 @@ def has_data(self): #} END interface + def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than @@ -160,7 +165,8 @@ def _closest_index(dcl, absofs): lo = mid + 1 # END handle bound # END for each delta absofs - return len(dcl)-1 + return len(dcl) - 1 + def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed @@ -173,6 +179,7 @@ def delta_list_apply(dcl, bbuf, write): delta_chunk_apply(dc, bbuf, write) # END for each dc + def delta_list_slice(dcl, absofs, size, ndcl): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. @@ -209,6 +216,7 @@ def delta_list_slice(dcl, absofs, size, ndcl): class DeltaChunkList(list): + """List with special functionality to deal with DeltaChunks. There are two types of lists we represent. The one was created bottom-up, working towards the latest delta, the other kind was created top-down, working from the @@ -252,16 +260,16 @@ def compress(self): dc = self[i] i += 1 if dc.data is None: - if first_data_index is not None and i-2-first_data_index > 1: - #if first_data_index is not None: + if first_data_index is not None and i - 2 - first_data_index > 1: + # if first_data_index is not None: nd = StringIO() # new data so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i-1): + for x in xrange(first_data_index, i - 1): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data - del(self[first_data_index:i-1]) + del(self[first_data_index:i - 1]) buf = nd.getvalue() self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) @@ -274,10 +282,10 @@ def compress(self): # END skip non-data chunks if first_data_index is None: - first_data_index = i-1 + first_data_index = i - 1 # END iterate list - #if slen_orig != len(self): + # if slen_orig != len(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self @@ -288,7 +296,7 @@ def check_integrity(self, target_size=-1): :raise AssertionError: if the size doen't match""" if target_size > -1: assert self[-1].rbound() == target_size - assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size # END target size verification if len(self) < 2: @@ -301,18 +309,19 @@ def check_integrity(self, target_size=-1): assert len(dc.data) >= dc.ts # END for each dc - left = islice(self, 0, len(self)-1) + left = islice(self, 0, len(self) - 1) right = iter(self) right.next() # this is very pythonic - we might have just use index based access here, # but this could actually be faster - for lft,rgt in izip(left, right): + for lft, rgt in izip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair class TopdownDeltaChunkList(DeltaChunkList): + """Represents a list which is generated by feeding its ancestor streams one by one""" __slots__ = tuple() @@ -356,19 +365,19 @@ def connect_with_next_base(self, bdcl): # END update target bounds if len(ccl) == 1: - self[dci-1] = ccl[0] + self[dci - 1] = ccl[0] else: # maybe try to compute the expenses here, and pick the right algorithm # It would normally be faster than copying everything physically though # TODO: Use a deque here, and decide by the index whether to extend # or extend left ! post_dci = self[dci:] - del(self[dci-1:]) # include deletion of dc + del(self[dci - 1:]) # include deletion of dc self.extend(ccl) self.extend(post_dci) slen = len(self) - dci += len(ccl)-1 # deleted dc, added rest + dci += len(ccl) - 1 # deleted dc, added rest # END handle chunk replacement # END for each chunk @@ -391,6 +400,7 @@ def is_loose_object(m): word = (b0 << 8) + b1 return b0 == 0x78 and (word % 31) == 0 + def loose_object_header_info(m): """ :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the @@ -402,6 +412,7 @@ def loose_object_header_info(m): return type_name, int(size) + def pack_object_header_info(data): """ :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) @@ -430,6 +441,7 @@ def pack_object_header_info(data): # end performance at expense of maintenance ... return (type_id, size, i) + def create_pack_object_header(obj_type, obj_size): """ :return: string defining the pack header comprised of the object type @@ -439,7 +451,7 @@ def create_pack_object_header(obj_type, obj_size): :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte if PY3: - hdr = bytearray() # output string + hdr = bytearray() # output string c = (obj_type << 4) | (obj_size & 0xf) obj_size >>= 4 @@ -447,10 +459,10 @@ def create_pack_object_header(obj_type, obj_size): hdr.append(c | 0x80) c = obj_size & 0x7f obj_size >>= 7 - #END until size is consumed + # END until size is consumed hdr.append(c) else: - hdr = bytes() # output string + hdr = bytes() # output string c = (obj_type << 4) | (obj_size & 0xf) obj_size >>= 4 @@ -458,11 +470,12 @@ def create_pack_object_header(obj_type, obj_size): hdr += chr(c | 0x80) c = obj_size & 0x7f obj_size >>= 7 - #END until size is consumed + # END until size is consumed hdr += chr(c) # end handle interpreter return hdr + def msb_size(data, offset=0): """ :return: tuple(read_bytes, size) read the msb size from the given random @@ -473,8 +486,8 @@ def msb_size(data, offset=0): hit_msb = False if PY3: while i < l: - c = data[i+offset] - size |= (c & 0x7f) << i*7 + c = data[i + offset] + size |= (c & 0x7f) << i * 7 i += 1 if not c & 0x80: hit_msb = True @@ -483,8 +496,8 @@ def msb_size(data, offset=0): # END while in range else: while i < l: - c = ord(data[i+offset]) - size |= (c & 0x7f) << i*7 + c = ord(data[i + offset]) + size |= (c & 0x7f) << i * 7 i += 1 if not c & 0x80: hit_msb = True @@ -494,7 +507,8 @@ def msb_size(data, offset=0): # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size + return i + offset, size + def loose_object_header(type, size): """ @@ -502,6 +516,7 @@ def loose_object_header(type, size): followed by the content stream of size 'size'""" return ('%s %i\0' % (force_text(type), size)).encode('ascii') + def write_object(type, size, read, write, chunk_size=chunk_size): """ Write the object as identified by type, size and source_stream into the @@ -522,6 +537,7 @@ def write_object(type, size, read, write, chunk_size=chunk_size): return tbw + def stream_copy(read, write, size, chunk_size): """ Copy a stream up to size bytes using the provided read and write methods, @@ -532,7 +548,7 @@ def stream_copy(read, write, size, chunk_size): # WRITE ALL DATA UP TO SIZE while True: - cs = min(chunk_size, size-dbw) + cs = min(chunk_size, size - dbw) # NOTE: not all write methods return the amount of written bytes, like # mmap.write. Its bad, but we just deal with it ... perhaps its not # even less efficient @@ -548,6 +564,7 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw + def connect_deltas(dstreams): """ Read the condensed delta chunk information from dstream and merge its information @@ -602,7 +619,7 @@ def connect_deltas(dstreams): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > base_size): + rbound > base_size): break dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) @@ -610,7 +627,7 @@ def connect_deltas(dstreams): elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i + c])) i += c tbw += c else: @@ -632,6 +649,7 @@ def connect_deltas(dstreams): return tdcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file @@ -678,11 +696,11 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size): + rbound > src_buf_size): break write(buffer(src_buf, cp_off, cp_size)) elif c: - write(db[i:i+c]) + write(db[i:i + c]) i += c else: raise ValueError("unexpected delta opcode 0") @@ -721,11 +739,11 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size): + rbound > src_buf_size): break write(buffer(src_buf, cp_off, cp_size)) elif c: - write(db[i:i+c]) + write(db[i:i + c]) i += c else: raise ValueError("unexpected delta opcode 0") @@ -749,7 +767,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): return False if canonical_length - binary_length and \ - (byte_ord(match[-1]) ^ byte_ord(sha1[len(match)-1])) & 0xf0: + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match) - 1])) & 0xf0: return False # END handle uneven canonnical length return True diff --git a/gitdb/pack.py b/gitdb/pack.py index 375cc59a9..b4ba7876c 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -72,8 +72,6 @@ __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') - - #{ Utilities def pack_object_at(cursor, offset, as_stream): @@ -107,7 +105,7 @@ def pack_object_at(cursor, offset, as_stream): total_rela_offset = i # REF DELTA elif type_id == REF_DELTA: - total_rela_offset = data_rela_offset+20 + total_rela_offset = data_rela_offset + 20 delta_info = data[data_rela_offset:total_rela_offset] # BASE OBJECT else: @@ -129,6 +127,7 @@ def pack_object_at(cursor, offset, as_stream): # END handle info # END handle stream + def write_stream_to_pack(read, write, zstream, base_crc=None): """Copy a stream as read from read function, zip it, and write the result. Count the number of written bytes and return it @@ -142,7 +141,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): crc = 0 if want_crc: crc = base_crc - #END initialize crc + # END initialize crc while True: chunk = read(chunk_size) @@ -153,18 +152,18 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): if want_crc: crc = crc32(compressed, crc) - #END handle crc + # END handle crc if len(chunk) != chunk_size: break - #END copy loop + # END copy loop compressed = zstream.flush() bw += len(compressed) write(compressed) if want_crc: crc = crc32(compressed, crc) - #END handle crc + # END handle crc return (br, bw, crc) @@ -173,6 +172,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): class IndexWriter(object): + """Utility to cache index information, allowing to write all information later in one go to the given stream **Note:** currently only writes v2 indices""" @@ -198,15 +198,15 @@ def write(self, pack_sha, write): sha_write(pack(">L", PackIndexFile.index_version_default)) # fanout - tmplist = list((0,)*256) # fanout or list with 64 bit offsets + tmplist = list((0,) * 256) # fanout or list with 64 bit offsets for t in self._objs: tmplist[byte_ord(t[0][0])] += 1 - #END prepare fanout + # END prepare fanout for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) - tmplist[i+1] += v - #END write each fanout entry + tmplist[i + 1] += v + # END write each fanout entry sha_write(pack('>L', tmplist[255])) # sha1 ordered @@ -215,8 +215,8 @@ def write(self, pack_sha, write): # crc32 for t in self._objs: - sha_write(pack('>L', t[1]&0xffffffff)) - #END for each crc + sha_write(pack('>L', t[1] & 0xffffffff)) + # END for each crc tmplist = list() # offset 32 @@ -224,15 +224,15 @@ def write(self, pack_sha, write): ofs = t[2] if ofs > 0x7fffffff: tmplist.append(ofs) - ofs = 0x80000000 + len(tmplist)-1 - #END hande 64 bit offsets - sha_write(pack('>L', ofs&0xffffffff)) - #END for each offset + ofs = 0x80000000 + len(tmplist) - 1 + # END hande 64 bit offsets + sha_write(pack('>L', ofs & 0xffffffff)) + # END for each offset # offset 64 for ofs in tmplist: sha_write(pack(">Q", ofs)) - #END for each offset + # END for each offset # trailer assert(len(pack_sha) == 20) @@ -242,8 +242,8 @@ def write(self, pack_sha, write): return sha - class PackIndexFile(LazyMixin): + """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -273,8 +273,9 @@ def _set_cache_(self, attr): self._cursor = mman.make_cursor(self._indexpath).use_region() # We will assume that the index will always fully fit into memory ! if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): - raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) - #END assert window size + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % ( + self._indexpath, self._cursor.file_size(), mman.window_size())) + # END assert window size else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -293,27 +294,25 @@ def _set_cache_(self, attr): setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) # END for each function to initialize - # INITIALIZE DATA # byte offset is 8 if version is 2, 0 otherwise self._initialize() # END handle attributes - #{ Access V1 def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" - return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] + return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0] def _sha_v1(self, i): """see ``_sha_v2``""" - base = 1024 + (i*24)+4 - return self._cursor.map()[base:base+20] + base = 1024 + (i * 24) + 4 + return self._cursor.map()[base:base + 20] def _crc_v1(self, i): """unsupported""" @@ -343,7 +342,7 @@ def _offset_v2(self, i): def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 - return self._cursor.map()[base:base+20] + return self._cursor.map()[base:base + 20] def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" @@ -369,7 +368,7 @@ def _read_fanout(self, byte_offset): out = list() append = out.append for i in xrange(256): - append(unpack_from('>L', d, byte_offset + i*4)[0]) + append(unpack_from('>L', d, byte_offset + i * 4)[0]) # END for each entry return out @@ -421,7 +420,7 @@ def sha_to_index(self, sha): get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: - lo = self._fanout_table[first_byte-1] + lo = self._fanout_table[first_byte - 1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # bisect until we have the sha @@ -455,7 +454,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: - lo = self._fanout_table[first_byte-1] + lo = self._fanout_table[first_byte - 1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes @@ -481,7 +480,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None if lo + 1 < self.size(): - next_sha = get_sha(lo+1) + next_sha = get_sha(lo + 1) if next_sha and next_sha == cur_sha: raise AmbiguousObjectName(partial_bin_sha) return lo @@ -500,6 +499,7 @@ def sha_to_index(self, sha): class PackFile(LazyMixin): + """A pack is a file written according to the Version 2 for git packs As we currently use memory maps, it could be assumed that the maximum size of @@ -516,7 +516,7 @@ class PackFile(LazyMixin): pack_version_default = 2 # offset into our data at which the first object starts - first_object_offset = 3*4 # header bytes + first_object_offset = 3 * 4 # header bytes footer_size = 20 # final sha def __init__(self, packpath): @@ -549,7 +549,6 @@ def _iter_objects(self, start_offset, as_stream=True): stream_copy(ostream.read, null.write, ostream.size, chunk_size) cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - # if a stream is requested, reset it beforehand # Otherwise return the Stream object directly, its derived from the # info object @@ -578,7 +577,7 @@ def data(self): def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] + return self._cursor.use_region(self._cursor.file_size() - 20).buffer()[:] def path(self): """:return: path to the packfile""" @@ -645,13 +644,14 @@ def stream_iter(self, start_offset=0): class PackEntity(LazyMixin): + """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - __slots__ = ( '_index', # our index file - '_pack', # our pack file - '_offset_map' # on demand dict mapping one offset to the next consecutive one - ) + __slots__ = ('_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) IndexFileCls = PackIndexFile PackFileCls = PackFile @@ -673,7 +673,7 @@ def _set_cache_(self, attr): offset_map = None if len(offsets_sorted) == 1: - offset_map = { offsets_sorted[0] : last_offset } + offset_map = {offsets_sorted[0]: last_offset} else: iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) @@ -895,10 +895,9 @@ def collect_streams(self, sha): :raise BadObject:""" return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - @classmethod def write_pack(cls, object_iter, pack_write, index_write=None, - object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + object_count=None, zlib_compression=zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. @@ -923,9 +922,9 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if not object_count: if not isinstance(object_iter, (tuple, list)): objs = list(object_iter) - #END handle list type + # END handle list type object_count = len(objs) - #END handle object + # END handle object pack_writer = FlexibleSha1Writer(pack_write) pwrite = pack_writer.write @@ -939,7 +938,7 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if wants_index: index = IndexWriter() - #END handle index header + # END handle index header actual_count = 0 for obj in objs: @@ -952,30 +951,31 @@ def write_pack(cls, object_iter, pack_write, index_write=None, crc = crc32(hdr) else: crc = None - #END handle crc + # END handle crc pwrite(hdr) # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc=crc) assert(br == obj.size) if wants_index: index.append(obj.binsha, crc, ofs) - #END handle index + # END handle index ofs += len(hdr) + bw if actual_count == object_count: break - #END abort once we are done - #END for each object + # END abort once we are done + # END for each object if actual_count != object_count: - raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) - #END count assertion + raise ValueError( + "Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + # END count assertion # write footer - pack_sha = pack_writer.sha(as_hex = False) + pack_sha = pack_writer.sha(as_hex=False) assert len(pack_sha) == 20 pack_write(pack_sha) ofs += len(pack_sha) # just for completeness ;) @@ -983,12 +983,12 @@ def write_pack(cls, object_iter, pack_write, index_write=None, index_sha = None if wants_index: index_sha = index.write(pack_sha, index_write) - #END handle index + # END handle index return pack_sha, index_sha @classmethod - def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.Z_BEST_SPEED): """Create a new on-disk entity comprised of a properly named pack file and a properly named and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files @@ -1012,5 +1012,4 @@ def create(cls, object_iter, base_dir, object_count = None, zlib_compression = z return cls(new_pack_path) - #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index b0a89002a..4478a0f44 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -38,14 +38,15 @@ except ImportError: pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', - 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', - 'FDStream', 'NullStream') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams class DecompressMemMapReader(LazyMixin): + """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly @@ -63,9 +64,9 @@ class DecompressMemMapReader(LazyMixin): to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', - '_cbr', '_phi') + '_cbr', '_phi') - max_read_size = 512*1024 # currently unused + max_read_size = 512 * 1024 # currently unused def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading @@ -214,7 +215,6 @@ def read(self, size=-1): return bytes() # END handle depletion - # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream @@ -263,7 +263,6 @@ def read(self, size=-1): self._cwe = cws + size # END handle tail - # if window is too small, make it larger so zip can decompress something if self._cwe - self._cws < 8: self._cwe = self._cws + 8 @@ -285,7 +284,7 @@ def read(self, size=-1): unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) - # end handle very special case ... + # end handle very special case ... self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) @@ -301,12 +300,13 @@ def read(self, size=-1): # to read, if we are called by compressed_bytes_read - it manipulates # us to empty the stream if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size-len(dcompdat)) + dcompdat += self.read(size - len(dcompdat)) # END handle special case return dcompdat class DeltaApplyReader(LazyMixin): + """A reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. @@ -332,15 +332,15 @@ class DeltaApplyReader(LazyMixin): * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( - "_bstream", # base stream to which to apply the deltas - "_dstreams", # tuple of delta stream readers - "_mm_target", # memory map of the delta-applied data - "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read - ) + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) #{ Configuration - k_max_memory_move = 250*1000*1000 + k_max_memory_move = 250 * 1000 * 1000 #} END configuration def __init__(self, stream_list): @@ -414,7 +414,6 @@ def _set_cache_brute_(self, attr): base_size = target_size = max(base_size, max_target_size) # END adjust buffer sizes - # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) @@ -440,11 +439,11 @@ def _set_cache_brute_(self, attr): ddata = allocate_memory(dstream.size - offset) ddata.write(dbuf) # read the rest from the stream. The size we give is larger than necessary - stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + stream_copy(dstream.read, ddata.write, dstream.size, 256 * mmap.PAGESIZE) ####################################################################### if 'c_apply_delta' in globals(): - c_apply_delta(bbuf, ddata, tbuf); + c_apply_delta(bbuf, ddata, tbuf) else: apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### @@ -463,7 +462,6 @@ def _set_cache_brute_(self, attr): self._mm_target = bbuf self._size = final_target_size - #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ @@ -512,13 +510,13 @@ def new(cls, stream_list): # END single object special handling if stream_list[-1].type_id in delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + raise ValueError( + "Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream return cls(stream_list) #} END interface - #{ OInfo like Interface @property @@ -543,6 +541,7 @@ def size(self): #{ W Streams class Sha1Writer(object): + """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" __slots__ = "sha1" @@ -565,7 +564,7 @@ def write(self, data): #{ Interface - def sha(self, as_hex = False): + def sha(self, as_hex=False): """:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise""" if as_hex: @@ -576,6 +575,7 @@ def sha(self, as_hex = False): class FlexibleSha1Writer(Sha1Writer): + """Writer producing a sha1 while passing on the written bytes to the given write function""" __slots__ = 'writer' @@ -590,8 +590,10 @@ def write(self, data): class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it and generates a sha""" __slots__ = ('buf', 'zip') + def __init__(self): Sha1Writer.__init__(self) self.buf = BytesIO() @@ -623,6 +625,7 @@ def getvalue(self): class FDCompressedSha1Writer(Sha1Writer): + """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor @@ -662,10 +665,12 @@ def close(self): class FDStream(object): + """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership""" __slots__ = ("_fd", '_pos') + def __init__(self, fd): self._fd = fd self._pos = 0 @@ -694,6 +699,7 @@ def close(self): class NullStream(object): + """A stream that does nothing but providing a stream interface. Use it like /dev/null""" __slots__ = tuple() diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index af6d9e0fd..528bcc144 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -32,7 +32,9 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') + class TestDBBase(TestBase): + """Base class providing testing routines on databases""" # data @@ -65,7 +67,6 @@ def _assert_object_writing_simple(self, db): assert len(shas) == db.size() assert len(shas[0]) == 20 - def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW **Note:** requires write access to the database""" @@ -126,4 +127,3 @@ def _assert_object_writing(self, db): assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode - diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index e141c2ba0..f96206744 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from gitdb.test.db.lib import ( - TestDBBase, + TestDBBase, fixture_path, with_rw_directory ) @@ -12,6 +12,7 @@ from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex + class TestGitDB(TestDBBase): def test_reading(self): @@ -28,8 +29,7 @@ def test_reading(self): assert gdb.size() >= ni sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() - sha_list = sha_list[:ni] # speed up tests ... - + sha_list = sha_list[:ni] # speed up tests ... # This is actually a test for compound functionality, but it doesn't # have a separate test module @@ -39,7 +39,7 @@ def test_reading(self): # mix even/uneven hexshas for i, binsha in enumerate(sha_list): - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha # END for each sha self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 1d6af9c99..024c194a2 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -3,13 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from gitdb.test.db.lib import ( - TestDBBase, + TestDBBase, with_rw_directory ) from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex + class TestLooseDB(TestDBBase): @with_rw_directory diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 97f721719..eb563c098 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -11,6 +11,7 @@ LooseObjectDB ) + class TestMemoryDB(TestDBBase): @with_rw_directory diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 963a71af7..a90158151 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -14,6 +14,7 @@ import os import random + class TestPackDB(TestDBBase): @with_rw_directory @@ -53,7 +54,6 @@ def test_writing(self, path): pdb.stream(sha) # END for each sha to query - # test short finding - be a bit more brutal here max_bytes = 19 min_bytes = 2 @@ -61,10 +61,10 @@ def test_writing(self, path): for i, sha in enumerate(sha_list): short_sha = sha[:max((i % max_bytes), min_bytes)] try: - assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha + assert pdb.partial_to_complete_sha(short_sha, len(short_sha) * 2) == sha except AmbiguousObjectName: num_ambiguous += 1 - pass # valid, we can have short objects + pass # valid, we can have short objects # END exception handling # END for each sha to find diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index db930827b..b774bafe4 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -3,8 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from gitdb.test.db.lib import ( - TestDBBase, - with_rw_directory, + TestDBBase, + with_rw_directory, fixture_path ) from gitdb.db import ReferenceDB @@ -16,6 +16,7 @@ import os + class TestReferenceDB(TestDBBase): def make_alt_file(self, alt_path, alt_list): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index d09b1cb8e..c4acd9218 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -24,6 +24,7 @@ #{ Bases class TestBase(unittest.TestCase): + """Base class for all tests""" @@ -49,6 +50,7 @@ def wrapper(self, *args, **kwargs): def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) @@ -78,6 +80,7 @@ def wrapper(self): def with_packs_rw(func): """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): src_pack_glob = fixture_path('packs/*') copy_files_globbed(src_pack_glob, path, hard_link_ok=True) @@ -91,12 +94,14 @@ def wrapper(self, path): #{ Routines + def fixture_path(relapath=''): """:return: absolute path into the fixture directory :param relapath: relative path into the fixtures directory, or '' to obtain the fixture directory itself""" return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): """Copy all files found according to the given source glob into the target directory :param hard_link_ok: if True, hard links will be created if possible. Otherwise @@ -127,11 +132,13 @@ def make_bytes(size_in_bytes, randomize=False): a = array('i', producer) return a.tostring() + def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata.encode("ascii") + data + def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" @@ -142,24 +149,27 @@ def make_memory_file(size_in_bytes, randomize=False): #{ Stream Utilities + class DummyStream(object): - def __init__(self): - self.was_read = False - self.bytes = 0 - self.closed = False - def read(self, size): - self.was_read = True - self.bytes = size + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False - def close(self): - self.closed = True + def read(self, size): + self.was_read = True + self.bytes = size - def _assert(self): - assert self.was_read + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read class DeriveTest(OStream): + def __init__(self, sha, type, size, stream, *args, **kwargs): self.myarg = kwargs.pop('myarg') self.args = args diff --git a/gitdb/test/performance/__init__.py b/gitdb/test/performance/__init__.py index 8b1378917..e69de29bb 100644 --- a/gitdb/test/performance/__init__.py +++ b/gitdb/test/performance/__init__.py @@ -1 +0,0 @@ - diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index ec45cf3a7..cbc52bc77 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -13,22 +13,17 @@ #} END invariants - -#{ Base Classes +#{ Base Classes class TestBigRepoR(TestBase): + """TestCase providing access to readonly 'big' repositories using the following member variables: - + * gitrepopath - + * read-only base path of the git source repository, i.e. .../git/.git""" - - #{ Invariants - head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' - head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' - #} END invariants - + def setUp(self): try: super(TestBigRepoR, self).setUp() @@ -37,11 +32,12 @@ def setUp(self): self.gitrepopath = os.environ.get(k_env_git_repo) if not self.gitrepopath: - logging.info("You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository") + logging.info( + "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", k_env_git_repo) ospd = os.path.dirname self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') # end assure gitrepo is set assert self.gitrepopath.endswith('.git') - + #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index e46031115..bdd2b0a37 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -6,7 +6,7 @@ from __future__ import print_function from gitdb.test.performance.lib import ( - TestBigRepoR + TestBigRepoR ) from gitdb import ( @@ -24,19 +24,20 @@ import os from time import time + class TestPackedDBPerformance(TestBigRepoR): - @skip_on_travis_ci + @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - + # sha lookup st = time() sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) - + # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info # END shuffle shas @@ -45,13 +46,14 @@ def test_pack_random_access(self): pdb_pack_info(sha) # END for each sha to look up elapsed = time() - st - + # discard cache del(pdb._entities) pdb.entities() - print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % + (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) # END for each random mode - + # query info and streams only max_items = 10000 # can wait longer when testing memory for pdb_fun in (pdb.info, pdb.stream): @@ -59,9 +61,10 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % + (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) # END for each function - + # retrieve stream and read all max_items = 5000 pdb_stream = pdb.stream @@ -74,8 +77,9 @@ def test_pack_random_access(self): total_size += stream.size elapsed = time() - st total_kib = total_size / 1000 - print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) - + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (max_items, total_kib, total_kib / elapsed, elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci def test_loose_correctness(self): """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back @@ -89,7 +93,7 @@ def test_loose_correctness(self): mdb = MemoryDB() for c, sha in enumerate(pdb.sha_iter()): ostream = pdb.stream(sha) - # the issue only showed on larger files which are hardly compressible ... + # the issue only showed on larger files which are hardly compressible ... if ostream.type != str_blob_type: continue istream = IStream(ostream.type, ostream.size, ostream.stream) @@ -101,7 +105,7 @@ def test_loose_correctness(self): if c and c % 1000 == 0: print("Verified %i loose object compression/decompression cycles" % c, file=sys.stderr) mdb._cache.clear() - # end for each sha to copy + # end for each sha to copy @skip_on_travis_ci def test_correctness(self): @@ -124,6 +128,6 @@ def test_correctness(self): # END for each index # END for each entity elapsed = time() - st - print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed), file=sys.stderr) + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % + (count, crc, elapsed, count / elapsed), file=sys.stderr) # END for each verify mode - diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index fe160ea54..f805e59fa 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -6,7 +6,7 @@ from __future__ import print_function from gitdb.test.performance.lib import ( - TestBigRepoR + TestBigRepoR ) from gitdb.db.pack import PackedDB @@ -18,27 +18,29 @@ import sys from time import time + class CountedNullStream(NullStream): __slots__ = '_bw' + def __init__(self): self._bw = 0 - + def bytes_written(self): return self._bw - + def write(self, d): self._bw += NullStream.write(self, d) - + class TestPackStreamingPerformance(TestBigRepoR): - + @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well ostream = CountedNullStream() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - + ni = 1000 count = 0 st = time() @@ -47,22 +49,23 @@ def test_pack_writing(self): pdb.stream(sha) if count == ni: break - #END gather objects for pack-writing + # END gather objects for pack-writing elapsed = time() - st - print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed), file=sys.stderr) - + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % + (ni, elapsed, ni / elapsed), file=sys.stderr) + st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 - print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) - - + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % + (total_kb, elapsed, total_kb / elapsed), sys.stderr) + @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - + # streaming only, meant for --with-profile runs ni = 5000 count = 0 @@ -78,5 +81,5 @@ def test_stream_reading(self): count += 1 elapsed = time() - st total_kib = total_size / 1000 - print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed), sys.stderr) - + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (ni, total_kib, total_kib / elapsed, elapsed, ni / elapsed), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 84c9dea3f..bd66b26ad 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -35,22 +35,22 @@ def read_chunked_stream(stream): # END read stream loop assert total == stream.size return stream - - + + #} END utilities class TestObjDBPerformance(TestBigRepoR): - - large_data_size_bytes = 1000*1000*50 # some MiB should do it - moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - - @skip_on_travis_ci + + large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it + moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB + + @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) string_ios = list() # list of streams we previously created - - # serial mode + + # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' print("Creating %s data ..." % desc, file=sys.stderr) @@ -59,32 +59,32 @@ def test_large_data_streaming(self, path): elapsed = time() - st print("Done (in %f s)" % elapsed, file=sys.stderr) string_ios.append(stream) - - # writing - due to the compression it will seem faster than it is + + # writing - due to the compression it will seem faster than it is st = time() sha = ldb.store(IStream('blob', size, stream)).binsha elapsed_add = time() - st assert ldb.has_object(sha) db_file = ldb.readable_db_object_path(bin_to_hex(sha)) fsize_kib = os.path.getsize(db_file) / 1000 - - + size_kib = size / 1000 - print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) - + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % + (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) + # reading all at once st = time() ostream = ldb.stream(sha) shadata = ostream.read() elapsed_readall = time() - st - + stream.seek(0) assert shadata == stream.getvalue() - print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) - - + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) + # reading in chunks of 1 MiB - cs = 512*1000 + cs = 512 * 1000 chunks = list() st = time() ostream = ldb.stream(sha) @@ -95,13 +95,14 @@ def test_large_data_streaming(self, path): break # END read in chunks elapsed_readchunks = time() - st - + stream.seek(0) assert b''.join(chunks) == stream.getvalue() - + cs_kib = cs / 1000 - print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) - + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) + # del db file so we keep something to do os.remove(db_file) # END for each randomization factor diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 578c29f73..519cdfdc1 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -4,10 +4,10 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from gitdb.test.lib import ( - TestBase, - DummyStream, - DeriveTest, - ) + TestBase, + DummyStream, + DeriveTest, +) from gitdb import ( OInfo, @@ -20,11 +20,11 @@ ) from gitdb.util import ( NULL_BIN_SHA - ) +) from gitdb.typ import ( str_blob_type - ) +) class TestBaseTypes(TestBase): @@ -54,7 +54,6 @@ def test_streams(self): assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) @@ -80,7 +79,7 @@ def test_streams(self): assert stream.bytes == 5 # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + DeriveTest(sha, str_blob_type, s, stream, 'mine', myarg=3)._assert() # test istream istream = IStream(str_blob_type, s, stream) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index aa43a093f..ed0a885bf 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" from gitdb.test.lib import ( - TestBase, + TestBase, fixture_path ) from gitdb import IStream @@ -12,6 +12,7 @@ from io import BytesIO + class TestExamples(TestBase): def test_base(self): diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 3ab2fec07..ff1057237 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -43,6 +43,7 @@ def bin_sha_from_filename(filename): return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) #} END utilities + class TestPack(TestBase): packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) @@ -50,8 +51,8 @@ class TestPack(TestBase): packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) - packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - + packfile_v2_3_ascii = ( + fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) def _assert_index_file(self, index, version, size): assert index.packfile_checksum() != index.indexfile_checksum() @@ -74,13 +75,12 @@ def _assert_index_file(self, index, version, size): assert entry[2] == index.crc(oidx) # verify partial sha - for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l], l*2) == oidx + for l in (4, 8, 11, 17, 20): + assert index.partial_sha_to_index(sha[:l], l * 2) == oidx # END for each object index in indexfile self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 assert pack.size() == size @@ -120,7 +120,6 @@ def _assert_pack_file(self, pack, version, size): dstream.seek(0) assert dstream.read() == data - # read chunks # NOTE: the current implementation is safe, it basically transfers # all calls to the underlying memory map @@ -128,7 +127,6 @@ def _assert_pack_file(self, pack, version, size): # END for each object assert num_obj == size - def test_pack_index(self): # check version 1 and 2 for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): @@ -146,9 +144,9 @@ def test_pack(self): @with_rw_directory def test_pack_entity(self, rw_dir): pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2), - (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + for packinfo, indexinfo in ((self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo indexfile, version, size = indexinfo entity = PackEntity(packfile) @@ -193,22 +191,23 @@ def test_pack_entity(self, rw_dir): pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 + def rewind_streams(): for obj in pack_objs: obj.stream.seek(0) - #END utility - for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + # END utility + for ppath, ipath, num_obj in zip((pack_path, ) * 2, (index_path, None), (len(pack_objs), None)): pfile = open(ppath, 'wb') iwrite = None if ipath: ifile = open(ipath, 'wb') iwrite = ifile.write - #END handle ip + # END handle ip # make sure we rewind the streams ... we work on the same objects over and over again if iteration > 0: rewind_streams() - #END rewind streams + # END rewind streams iteration += 1 pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) @@ -230,8 +229,8 @@ def rewind_streams(): assert idx.packfile_checksum() == pack_sha assert idx.indexfile_checksum() == index_sha assert idx.size() == len(pack_objs) - #END verify files exist - #END for each packpath, indexpath pair + # END verify files exist + # END for each packpath, indexpath pair # verify the packs throughly rewind_streams() @@ -242,10 +241,9 @@ def rewind_streams(): for use_crc in range(2): assert entity.is_valid_stream(info.binsha, use_crc) # END for each crc mode - #END for each info + # END for each info assert count == len(pack_objs) - def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 44a557d53..96268252d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -31,10 +31,12 @@ import os from io import BytesIO + class TestStream(TestBase): + """Test stream classes""" - data_sizes = (15, 10000, 1000*1024+512) + data_sizes = (15, 10000, 1000 * 1024 + 512) def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): """Make stream tests - the orig_stream is seekable, allowing it to be @@ -43,13 +45,13 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): :param rewind_stream: function called to rewind the stream to make it ready for reuse""" ns = 10 - assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + assert len(cdata) > ns - 1, "Data must be larger than %i, was %i" % (ns, len(cdata)) # read in small steps ss = len(cdata) // ns for i in range(ns): data = stream.read(ss) - chunk = cdata[i*ss:(i+1)*ss] + chunk = cdata[i * ss:(i + 1) * ss] assert data == chunk # END for each step rest = stream.read() @@ -136,7 +138,7 @@ def test_compressed_writer(self): self.failUnlessRaises(OSError, os.close, fd) # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) assert len(written_data) == os.path.getsize(path) os.close(fd) @@ -156,7 +158,7 @@ def test_decompress_reader_special_case(self): data = ostream.read() assert len(data) == ostream.size - # Putting it back in should yield nothing new - after all, we have + # Putting it back in should yield nothing new - after all, we have dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) assert dump.hexsha == sha # end for each loose object sha to test diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index e79355aaf..1dee54461 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -16,6 +16,7 @@ class TestUtils(TestBase): + def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 @@ -73,7 +74,6 @@ def test_lockedfd(self): del(lfd) assert not os.path.isfile(lockfilepath) - # write data - concurrently lfd = LockedFD(my_file) olfd = LockedFD(my_file) diff --git a/gitdb/typ.py b/gitdb/typ.py index bc7ba5828..98d15f3ec 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -str_blob_type = b'blob' +str_blob_type = b'blob' str_commit_type = b'commit' -str_tree_type = b'tree' -str_tag_type = b'tag' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/util.py b/gitdb/util.py index 93ba7f0ec..5b451faad 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -11,10 +11,10 @@ from io import StringIO from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer +) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -22,7 +22,7 @@ mman = StaticWindowMapManager() else: mman = SlidingWindowMapManager() -#END handle mman +# END handle mman import hashlib @@ -31,6 +31,7 @@ except ImportError: from struct import unpack, calcsize __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): try: size = __calcsize_cache[fmt] @@ -38,7 +39,7 @@ def unpack_from(fmt, data, offset=0): size = calcsize(fmt) __calcsize_cache[fmt] = size # END exception handling - return unpack(fmt, data[offset : offset + size]) + return unpack(fmt, data[offset: offset + size]) # END own unpack_from implementation @@ -67,8 +68,8 @@ def unpack_from(fmt, data, offset=0): fsync = os.fsync # Backwards compatibility imports -from gitdb.const import ( - NULL_BIN_SHA, +from gitdb.const import ( + NULL_BIN_SHA, NULL_HEX_SHA ) @@ -76,7 +77,9 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... + class _RandomAccessStringIO(object): + """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' @@ -96,6 +99,7 @@ def __getitem__(self, i): def __getslice__(self, start, end): return self.getvalue()[start:end] + def byte_ord(b): """ Return the integer representation of the byte string. This supports Python @@ -110,6 +114,7 @@ def byte_ord(b): #{ Routines + def make_sha(source=''.encode("ascii")): """A python2.4 workaround for the sha/hashlib module fiasco @@ -121,6 +126,7 @@ def make_sha(source=''.encode("ascii")): sha1 = sha.sha(source) return sha1 + def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" if size == 0: @@ -134,7 +140,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return _RandomAccessStringIO("\0"*size) + return _RandomAccessStringIO("\0" * size) # END handle memory allocation @@ -166,6 +172,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): return _RandomAccessStringIO(contents) return contents + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible @@ -178,25 +185,28 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): **Note** for now we don't try to use O_NOATIME directly as the right value needs to be shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + fd = os.open(filepath, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) try: return file_contents_ro(fd, stream, allow_mmap) finally: close(fd) # END assure file is closed + def sliding_ro_buffer(filepath, flags=0): """ :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: return sha return bin_to_hex(sha) + def to_bin_sha(sha): if len(sha) == 20: return sha @@ -209,6 +219,7 @@ def to_bin_sha(sha): #{ Utilities class LazyMixin(object): + """ Base class providing an interface to lazily retrieve attribute values upon first access. If slots are used, memory will only be reserved once the attribute @@ -240,6 +251,7 @@ def _set_cache_(self, attr): class LockedFD(object): + """ This class facilitates a safe read and write operation to a file on disk. If we write to 'file', we obtain a lock file at 'file.lock' and write to @@ -290,7 +302,7 @@ def open(self, write=False, stream=False): # try to open the lock file binary = getattr(os, 'O_BINARY', 0) - lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) if not write: diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index a2640fd23..c08cab555 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -24,7 +24,7 @@ def buffer(obj, offset, size=None): return obj[offset:] else: # return memoryview(obj)[offset:offset+size] - return obj[offset:offset+size] + return obj[offset:offset + size] # end buffer reimplementation memoryview = memoryview diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 2d03ad30c..5855062bf 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -7,6 +7,7 @@ string_types = (basestring, ) text_type = unicode + def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -16,6 +17,7 @@ def force_bytes(data, encoding="ascii"): return data + def force_text(data, encoding="utf-8"): if isinstance(data, text_type): return data diff --git a/setup.py b/setup.py index 4f8d1d5b2..6c67dc6aa 100755 --- a/setup.py +++ b/setup.py @@ -1,70 +1,74 @@ #!/usr/bin/env python -from distutils.core import setup, Extension +from distutils.core import setup, Extension from distutils.command.build_py import build_py from distutils.command.build_ext import build_ext -import os, sys +import os +import sys -# wow, this is a mixed bag ... I am pretty upset about all of this ... +# wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None try: - # don't pull it in if we don't have to - if 'setuptools' in sys.modules: - import setuptools.command.build_py as setuptools_build_py_module - from setuptools.command.build_ext import build_ext + # don't pull it in if we don't have to + if 'setuptools' in sys.modules: + import setuptools.command.build_py as setuptools_build_py_module + from setuptools.command.build_ext import build_ext except ImportError: - pass + pass + class build_ext_nofail(build_ext): - """Doesn't fail when build our optional extensions""" - def run(self): - try: - build_ext.run(self) - except Exception: - print("Ignored failure when building extensions, pure python modules will be used instead") - # END ignore errors - + + """Doesn't fail when build our optional extensions""" + + def run(self): + try: + build_ext.run(self) + except Exception: + print("Ignored failure when building extensions, pure python modules will be used instead") + # END ignore errors + def get_data_files(self): - """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, - the line dealing with the ``plen`` has a bug which causes it to truncate too much. - It is fixed in the system interpreters as they receive patches, and shows how - bad it is if something doesn't have proper unittests. - The code here is a plain copy of the python2.6 version which works for all. - - Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - - # this one is just for the setup tools ! They don't iniitlialize this variable - # when they should, but do it on demand using this method.Its crazy - if hasattr(self, 'analyze_manifest'): - self.analyze_manifest() - # END handle setuptools ... - - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir)+1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - + """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, + the line dealing with the ``plen`` has a bug which causes it to truncate too much. + It is fixed in the system interpreters as they receive patches, and shows how + bad it is if something doesn't have proper unittests. + The code here is a plain copy of the python2.6 version which works for all. + + Generate list of '(package,src_dir,build_dir,filenames)' tuples""" + data = [] + if not self.packages: + return data + + # this one is just for the setup tools ! They don't iniitlialize this variable + # when they should, but do it on demand using this method.Its crazy + if hasattr(self, 'analyze_manifest'): + self.analyze_manifest() + # END handle setuptools ... + + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir) + 1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + build_py.get_data_files = get_data_files if setuptools_build_py_module: - setuptools_build_py_module.build_py._get_data_files = get_data_files + setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too # NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot @@ -76,15 +80,15 @@ def get_data_files(self): version_info = (0, 6, 1) __version__ = '.'.join(str(i) for i in version_info) -setup(cmdclass={'build_ext':build_ext_nofail}, - name = "gitdb", - version = __version__, - description = "Git Object Database", - author = __author__, - author_email = __contact__, - url = __homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), - package_dir = {'gitdb':'gitdb'}, +setup(cmdclass={'build_ext': build_ext_nofail}, + name="gitdb", + version=__version__, + description="Git Object Database", + author=__author__, + author_email=__contact__, + url=__homepage__, + packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), + package_dir = {'gitdb': 'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, @@ -94,26 +98,26 @@ def get_data_files(self): # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", - # "Development Status :: 4 - Beta", - "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Operating System :: POSIX", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - ],) + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", +],) From c1998a074d2fd1773322e4595f30a5ecbbd54e32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 15:23:00 +0100 Subject: [PATCH 0287/1790] Fixed python 3 performance regression It makes the difference between tests in 110s, or 11s --- doc/source/changes.rst | 5 +++++ smmap/__init__.py | 2 +- smmap/util.py | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d5ed8e378..6cf9c83fa 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +********** +v0.8.4 +********** +- Fixed Python 3 performance regression + ********** v0.8.3 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index c494648d7..5f0e095cc 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 3) +version_info = (0, 8, 4) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/util.py b/smmap/util.py index e079a527a..6daa1fa12 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,9 +23,10 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - # return memoryview(obj)[offset:offset+size] + # Actually, for gitpython this is fastest ... . + return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! - return obj[offset:offset + size] + # return obj[offset:offset + size] def string_types(): From b1c9d3eb5b13f2feecb242701f5b4842184f6234 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 15:28:56 +0100 Subject: [PATCH 0288/1790] A minor fix after porting git-python over to PY3 It doesn't do anything (in terms of fixing an issue), but it should be more correct than what was there previously --- gitdb/utils/encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 5855062bf..4d270af9d 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -22,7 +22,7 @@ def force_text(data, encoding="utf-8"): if isinstance(data, text_type): return data - if isinstance(data, string_types): + if isinstance(data, bytes): return data.decode(encoding) if compat.PY3: From e0b0becd97afb9b7ac434c5fabdadd20070d643d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 15:55:09 +0100 Subject: [PATCH 0289/1790] Restore compatibility to python 3.0 to 3.4 --- doc/source/changes.rst | 5 +++++ smmap/__init__.py | 2 +- smmap/buf.py | 37 +++++++++++++++++++++++++++---------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6cf9c83fa..ec423694c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +********** +v0.8.5 +********** +- Fixed Python 3.0-3.3 regression, which also causes smmap to become about 3 times slower depending on the code path. It's related to this bug (http://bugs.python.org/issue15958), which was fixed in python 3.4 + ********** v0.8.4 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 5f0e095cc..46b0002e4 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 4) +version_info = (0, 8, 5) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index 17d2d369b..b3b71c49c 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -3,6 +3,8 @@ __all__ = ["SlidingWindowMapBuffer"] +import sys + try: bytes except NameError: @@ -79,16 +81,31 @@ def __getslice__(self, i, j): ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code - md = list() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - md.append(d) - # END while there are bytes to read - return bytes().join(md) + pyvers = sys.version_info[:2] + if (3, 0) <= pyvers <= (3, 3): + # Memory view cannot be joined below python 3.4 ... + out = bytes() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # This is slower than the join ... but what can we do ... + out += d + # END while there are bytes to read + return out + else: + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + md.append(d) + # END while there are bytes to read + return bytes().join(md) # END fast or slow path #{ Interface From fdc1d68b01f0d5dd601cdcc29df0eee19787d7c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 17:02:35 +0100 Subject: [PATCH 0290/1790] Fixed python 3 compatibility issue that only showed on windows And bumped version to 0.6.2 --- gitdb/__init__.py | 2 +- gitdb/util.py | 12 ++++++------ setup.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 791a2ef2f..020a5795f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 1) +version_info = (0, 6, 2) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/util.py b/gitdb/util.py index 5b451faad..8f80156b2 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,7 +8,7 @@ import sys import errno -from io import StringIO +from io import BytesIO from smmap import ( StaticWindowMapManager, @@ -78,14 +78,14 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... -class _RandomAccessStringIO(object): +class _RandomAccessBytesIO(object): """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' def __init__(self, buf=''): - self._sio = StringIO(buf) + self._sio = BytesIO(buf) def __getattr__(self, attr): return getattr(self._sio, attr) @@ -130,7 +130,7 @@ def make_sha(source=''.encode("ascii")): def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" if size == 0: - return _RandomAccessStringIO('') + return _RandomAccessBytesIO(b'') # END handle empty chunks gracefully try: @@ -140,7 +140,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return _RandomAccessStringIO("\0" * size) + return _RandomAccessBytesIO(b"\0" * size) # END handle memory allocation @@ -169,7 +169,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: - return _RandomAccessStringIO(contents) + return _RandomAccessBytesIO(contents) return contents diff --git a/setup.py b/setup.py index 6c67dc6aa..12c936a17 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 1) +version_info = (0, 6, 2) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext': build_ext_nofail}, @@ -92,8 +92,8 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('smmap (>=0.8.3)', ), - install_requires=('smmap >= 0.8.3'), + requires=('smmap (>=0.8.5)', ), + install_requires=('smmap >= 0.8.5'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From cb72f81e1407a86d85215a7fba4c2905c2451e0c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:17:23 +0100 Subject: [PATCH 0291/1790] Added coverage configuration Adjusted sublime project too --- .coveragerc | 11 +++++++++++ .gitignore | 3 ++- .travis.yml | 2 +- etc/sublime-text/gitdb.sublime-project | 24 +++++++----------------- 4 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..45d72abc9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,11 @@ +[run] +source = gitdb + +; to make nosetests happy +[report] +omit = + */smmap/* + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index c6247dbb0..e0b4e8579 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ MANIFEST +.coverage build/ dist/ *.pyc *.o *.so .noseids -*.sublime-workspace \ No newline at end of file +*.sublime-workspace diff --git a/.travis.yml b/.travis.yml index 761edc19b..d436229f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ git: install: - pip install coveralls script: - - nosetests -v + - nosetests -v --with-coverage after_success: - coveralls diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project index bc0e37f0a..d0e2e5132 100644 --- a/etc/sublime-text/gitdb.sublime-project +++ b/etc/sublime-text/gitdb.sublime-project @@ -15,7 +15,10 @@ "folder_exclude_patterns" : [ ".git", "cover", - "gitdb/ext" + "gitdb/ext", + "dist", + "doc/build", + ".tox" ] }, // SMMAP @@ -32,22 +35,9 @@ "folder_exclude_patterns" : [ ".git", "cover", - ] - }, - // ASYNC - //////// - { - "follow_symlinks": true, - "path": "../../gitdb/ext/async", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", + "dist", + "doc/build", + ".tox" ] }, ] From de96c522ff20fa99d13128784a393b619dd0b33b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:26:18 +0100 Subject: [PATCH 0292/1790] Fixed yet another issue with smmap's latest changes Now we deal with memory views as well ... --- gitdb/pack.py | 9 +++++++-- gitdb/utils/compat.py | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index b4ba7876c..d2666d601 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -62,7 +62,12 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import izip, buffer, xrange +from gitdb.utils.compat import ( + izip, + buffer, + xrange, + to_bytes +) import tempfile import array @@ -864,7 +869,7 @@ def collect_streams_at_offset(self, offset): stream = streams[-1] while stream.type_id in delta_types: if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(stream.delta_info) + sindex = self._index.sha_to_index(to_bytes(stream.delta_info)) if sindex is None: break stream = self._pack.stream(self._index.offset(sindex)) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index c08cab555..a7899cb14 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -15,6 +15,9 @@ # Python 2 buffer = buffer memoryview = buffer + # Assume no memory view ... + def to_bytes(i): + return i except NameError: # Python 3 has no `buffer`; only `memoryview` # However, it's faster to just slice the object directly, maybe it keeps a view internally @@ -26,6 +29,11 @@ def buffer(obj, offset, size=None): # return memoryview(obj)[offset:offset+size] return obj[offset:offset + size] # end buffer reimplementation + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + def to_bytes(i): + if isinstance(i, memoryview): + return i.tobytes() + return i memoryview = memoryview From a64e64a9735e8067ca196ab19711d136bd9b309c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:27:22 +0100 Subject: [PATCH 0293/1790] Bumped version to 0.6.3 --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 020a5795f..2ba8725eb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 2) +version_info = (0, 6, 3) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 12c936a17..e634e37f2 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 2) +version_info = (0, 6, 3) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext': build_ext_nofail}, From 9b3a34b7d00285cf9028d19e82de6b155d0096c7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:53:52 +0100 Subject: [PATCH 0294/1790] Improved coverage configuration --- .coveragerc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 45d72abc9..71b6ef701 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,9 +3,5 @@ source = gitdb ; to make nosetests happy [report] -omit = - */smmap/* - */yaml* - */tests/* - */python?.?/* - */site-packages/nose/* \ No newline at end of file +include = */gitdb/* +omit = */gitdb/ext/* From 70fae1f98bb7d44b58d94a183e9eb8b590bc23bf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 16:01:06 +0100 Subject: [PATCH 0295/1790] Initial attempt to fix resource usage Reference counting is now done manually, but it seems that things can still go wrong at least during testing --- doc/source/changes.rst | 6 ++++++ smmap/mman.py | 27 +++++++++++++-------------- smmap/test/test_mman.py | 26 +++++++++++++------------- smmap/test/test_util.py | 9 +-------- smmap/util.py | 33 +++++++++++++++++++-------------- 5 files changed, 52 insertions(+), 49 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ec423694c..f9f328757 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +********** +v0.8.6 +********** +- Fixed issue with resources never being freed as mmaps were never closed. +- Client counting is now done manually, instead of relying on pyton's reference count + ********** v0.8.5 ********** diff --git a/smmap/mman.py b/smmap/mman.py index c7a459558..6e8b9eeaf 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -8,7 +8,6 @@ buffer, ) -from weakref import ref import sys from functools import reduce @@ -55,8 +54,7 @@ def _destroy(self): # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted try: - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: + if len(self._rlist) == 0: # Free all resources associated with the mapped file self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager @@ -78,7 +76,7 @@ def _copy_from(self, rhs): self._size = rhs._size if self._region is not None: - self._region.increment_usage_count() + self._region.increment_client_count() # END handle regions def __copy__(self): @@ -126,20 +124,22 @@ def use_region(self, offset=0, size=0, flags=0): if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) + self._region.increment_client_count() # END need region handling - self._region.increment_usage_count() self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) return self def unuse_region(self): - """Unuse the ucrrent region. Does nothing if we have no current region + """Unuse the current region. Does nothing if we have no current region **Note:** the cursor unuses the region automatically upon destruction. It is recommended to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" + if self._region is not None: + self._region.increment_client_count(-1) self._region = None # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! @@ -184,12 +184,10 @@ def size(self): """:return: amount of bytes we point to""" return self._size - def region_ref(self): - """:return: weak ref to our mapped region. + def region(self): + """:return: our mapped region, or None if nothing is mapped yet :raise AssertionError: if we have no current region. This is only useful for debugging""" - if self._region is None: - raise AssertionError("region not set") - return ref(self._region) + return self._region def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors @@ -311,8 +309,8 @@ def _collect_lru_region(self, size): lru_list = None for regions in self._fdict.values(): for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count() - 2 == 0 and + # check client count - if it's 1, it's just us + if (region.client_count() == 1 and (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions @@ -326,6 +324,7 @@ def _collect_lru_region(self, size): num_found += 1 del(lru_list[lru_list.index(lru_region)]) + lru_region.increment_client_count(-1) self._memory_size -= lru_region.size() self._handle_count -= 1 # END while there is more memory to free @@ -449,7 +448,7 @@ def force_map_handle_removal_win(self, base_path): for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: - region._mf.close() + region.release() num_closed += 1 # END path matches # END for each path diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index b718b065f..c8b9c703e 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -119,21 +119,21 @@ def test_memman_operation(self): # window size is 0 for static managers, hence size will be 0. We take that into consideration size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us + rr = c.region() + assert rr.client_count() == 2 # the manager and the cursor and us assert man.num_open_files() == 1 assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() + assert man.mapped_memory_size() == rr.size() # assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr.ofs_begin() == 0 # it was aligned and expanded if man.window_size(): # but isn't larger than the max window (aligned) - assert rr().size() == align_to_mmap(man.window_size(), True) + assert rr.size() == align_to_mmap(man.window_size(), True) else: - assert rr().size() == fc.size + assert rr.size() == fc.size # END ignore static managers which dont use windows and are aligned to file boundaries assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] @@ -141,7 +141,7 @@ def test_memman_operation(self): # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 assert c.use_region(0, nsize).is_valid() - assert c.region_ref()() == rr() + assert c.region() == rr assert man.num_file_handles() == 1 assert c.size() == nsize assert c.ofs_begin() == 0 @@ -154,15 +154,15 @@ def test_memman_operation(self): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager + assert c.region() is not rr # old region is still available, but has not curser ref anymore + assert rr.client_count() == 1 # only held by manager else: assert c.size() < fc.size # END ignore static managers which only have one handle per file - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file + rr = c.region() + assert rr.client_count() == 2 # manager + cursor + assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr.ofs_end() <= fc.size # it cannot be larger than the file assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # unising a region makes the cursor invalid diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 0bbf91b65..0a162607e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -88,12 +88,7 @@ def test_region(self): # auto-refcount assert rfull.client_count() == 1 rfull2 = rfull - assert rfull.client_count() == 2 - - # usage - assert rfull.usage_count() == 0 - rfull.increment_usage_count() - assert rfull.usage_count() == 1 + assert rfull.client_count() == 1, "no auto-counting" # window constructor w = MapWindow.from_region(rfull) @@ -106,8 +101,6 @@ def test_region_list(self): for item in (fc.path, fd): ml = MapRegionList(item) - assert ml.client_count() == 1 - assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size diff --git a/smmap/util.py b/smmap/util.py index 6daa1fa12..1d20e5040 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -177,6 +177,8 @@ def __init__(self, path_or_fd, ofs, size, flags=0): os.close(fd) # END only close it if we opened it # END close file handle + # We assume the first one to use us keeps us around + self.increment_client_count() def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ @@ -222,17 +224,25 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" - from sys import getrefcount - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self) - 3 - - def usage_count(self): - """:return: amount of usages so far""" return self._uc - def increment_usage_count(self): - """Adjust the usage count by the given positive or negative offset""" - self._uc += 1 + def increment_client_count(self, ofs = 1): + """Adjust the usage count by the given positive or negative offset. + If usage count equals 0, we will auto-release our resources + :return: True if we released resources, False otherwise. In the latter case, we can still be used""" + self._uc += ofs + assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc + + if self.client_count() == 0: + self.release() + return True + else: + return False + # end handle release + + def release(self): + """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" + self._mf.close() # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: @@ -268,11 +278,6 @@ def __init__(self, path_or_fd): self._path_or_fd = path_or_fd self._file_size = None - def client_count(self): - """:return: amount of clients which hold a reference to this instance""" - from sys import getrefcount - return getrefcount(self) - 3 - def path_or_fd(self): """:return: path or file descriptor we are attached to""" return self._path_or_fd From f071ffdcacbafff648cd29d6f75fe27c47f53210 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 16:57:50 +0100 Subject: [PATCH 0296/1790] All tests work, bumped version --- doc/source/changes.rst | 2 +- smmap/__init__.py | 2 +- smmap/buf.py | 5 +++++ smmap/mman.py | 7 +++++-- smmap/test/test_buf.py | 3 ++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f9f328757..f99e85fb7 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ********** -v0.8.6 +v0.9.0 ********** - Fixed issue with resources never being freed as mmaps were never closed. - Client counting is now done manually, instead of relying on pyton's reference count diff --git a/smmap/__init__.py b/smmap/__init__.py index 46b0002e4..e711bbbf3 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 5) +version_info = (0, 9, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index b3b71c49c..e6f24341d 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -93,6 +93,7 @@ def __getslice__(self, i, j): l -= len(d) # This is slower than the join ... but what can we do ... out += d + del(d) # END while there are bytes to read return out else: @@ -103,6 +104,10 @@ def __getslice__(self, i, j): d = c.buffer()[:l] ofs += len(d) l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() md.append(d) # END while there are bytes to read return bytes().join(md) diff --git a/smmap/mman.py b/smmap/mman.py index 6e8b9eeaf..7180c75b3 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -58,7 +58,7 @@ def _destroy(self): # Free all resources associated with the mapped file self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager - except TypeError: + except (TypeError, KeyError): # sometimes, during shutdown, getrefcount is None. Its possible # to re-import it, however, its probably better to just ignore # this python problem (for now). @@ -70,11 +70,14 @@ def _destroy(self): def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" self._manager = rhs._manager - self._rlist = rhs._rlist + self._rlist = type(rhs._rlist)(rhs._rlist) self._region = rhs._region self._ofs = rhs._ofs self._size = rhs._size + for region in self._rlist: + region.increment_client_count() + if self._region is not None: self._region.increment_client_count() # END handle regions diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 03377154b..984b43254 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -12,7 +12,6 @@ from time import time import sys import os -import logging man_optimal = SlidingWindowMapManager() @@ -104,6 +103,7 @@ def test_basics(self): assert len(d) == ofs_end - ofs_start assert d == data[ofs_start:ofs_end] num_bytes += len(d) + del d else: pos = randint(0, fsize) assert buf[pos] == data[pos] @@ -122,6 +122,7 @@ def test_basics(self): % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), file=sys.stderr) # END handle access mode + del buf # END for each manager # END for each input os.close(fd) From a38efa84daef914e4de58d1905a500d8d14aaf45 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 18:03:33 +0100 Subject: [PATCH 0297/1790] Artificially restrict test-runs to assure we don't leak handles --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index cb0c16e44..b967f502e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ python: install: - pip install coveralls script: + - ulimit -n 48 + - ulimit -n - nosetests --with-coverage after_success: - coveralls From 560a211001064261eb25ca874980591790fb7986 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 18:09:45 +0100 Subject: [PATCH 0298/1790] Fixed possible file-handle leak Configured travis to artificially restrict handle count to protect from regression in that regard --- .travis.yml | 2 ++ gitdb/stream.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d436229f0..8cab8225f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ git: install: - pip install coveralls script: + - ulimit -n 48 + - ulimit -n - nosetests -v --with-coverage after_success: - coveralls diff --git a/gitdb/stream.py b/gitdb/stream.py index 4478a0f44..d855257cd 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -91,9 +91,7 @@ def _set_cache_(self, attr): self._parse_header_info() def __del__(self): - if self._close: - self._m.close() - # END handle resource freeing + self.close() def _parse_header_info(self): """If this stream contains object data, parse the header info and skip the @@ -141,6 +139,16 @@ def data(self): """:return: random access compatible data we are working on""" return self._m + def close(self): + """Close our underlying stream of compressed bytes if this was allowed during initialization + :return: True if we closed the underlying stream + :note: can be called safely + """ + if self._close: + self._m.close() + self._close = False + # END handle resource freeing + def compressed_bytes_read(self): """ :return: number of compressed bytes read. This includes the bytes it From be294278a0087f21d565a1084fb220ff936ae0bd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 19:57:19 +0100 Subject: [PATCH 0299/1790] Protected stream closure against possibilty of being a bytes For some reason, it gets bytes where it did expect a stream ... . Probably I should have figured out where this was input, instead of fixing it the brutal way --- gitdb/db/loose.py | 3 ++- gitdb/stream.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index e924080ef..4732b56da 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -159,7 +159,8 @@ def info(self, sha): typ, size = loose_object_header_info(m) return OInfo(sha, typ, size) finally: - m.close() + if hasattr(m, 'close'): + m.close() # END assure release of system resources def stream(self, sha): diff --git a/gitdb/stream.py b/gitdb/stream.py index d855257cd..826def387 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -145,7 +145,8 @@ def close(self): :note: can be called safely """ if self._close: - self._m.close() + if hasattr(self._m, 'close'): + self._m.close() self._close = False # END handle resource freeing From 7bde7b098b07291227fcbc4eb900ebf13c9191a2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jan 2015 17:34:53 +0100 Subject: [PATCH 0300/1790] Fixed up tests to use the GITDB_TEST_GIT_REPO_BASE at all times I have verified that all tests are working, even without a parent git repository, as long as the said environment variable is set. Fixes #16 --- gitdb/exc.py | 12 +++++++----- gitdb/test/db/test_git.py | 3 ++- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 29 ++++++++++++++++++++++++++++- gitdb/test/performance/lib.py | 30 ++---------------------------- gitdb/test/test_example.py | 8 +++----- 6 files changed, 43 insertions(+), 41 deletions(-) diff --git a/gitdb/exc.py b/gitdb/exc.py index d58442f2b..817ac7b62 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -12,12 +12,10 @@ class ODBError(Exception): class InvalidDBRoot(ODBError): - """Thrown if an object database cannot be initialized at the given path""" class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the failed sha""" @@ -25,19 +23,23 @@ def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) -class ParseError(ODBError): +class BadName(ODBError): + """A name provided to rev_parse wasn't understood""" + + def __str__(self): + return "Ref '%s' did not resolve to an object" % self.args[0] + +class ParseError(ODBError): """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): - """Thrown if a possibly shortened name does not uniquely represent a single object in the database""" class BadObjectType(ODBError): - """The object had an unsupported type""" diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index f96206744..f28ffb761 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,6 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php +import os from gitdb.test.db.lib import ( TestDBBase, fixture_path, @@ -16,7 +17,7 @@ class TestGitDB(TestDBBase): def test_reading(self): - gdb = GitDB(fixture_path('../../../.git/objects')) + gdb = GitDB(os.path.join(self.gitrepopath, 'objects')) # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index b774bafe4..25cf37d1b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -40,7 +40,7 @@ def test_writing(self, path): # setup alternate file # add two, one is invalid - own_repo_path = fixture_path('../../../.git/objects') # use own repo + own_repo_path = os.path.join(self.gitrepopath, 'objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index c4acd9218..a089eac6f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -18,14 +18,41 @@ import shutil import os import gc +import logging from functools import wraps #{ Bases class TestBase(unittest.TestCase): + """Base class for all tests - """Base class for all tests""" + TestCase providing access to readonly repositories using the following member variables. + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git + """ + + #{ Invvariants + k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" + #} END invariants + + @classmethod + def setUpClass(cls): + try: + super(TestBase, cls).setUpClass() + except AttributeError: + pass + + cls.gitrepopath = os.environ.get(cls.k_env_git_repo) + if not cls.gitrepopath: + logging.info( + "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", cls.k_env_git_repo) + ospd = os.path.dirname + cls.gitrepopath = os.path.join(ospd(ospd(ospd(__file__))), '.git') + # end assure gitrepo is set + assert cls.gitrepopath.endswith('.git') #} END bases diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index cbc52bc77..fa4dd209d 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -3,41 +3,15 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" -import os -import logging from gitdb.test.lib import TestBase -#{ Invvariants -k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" -#} END invariants - #{ Base Classes class TestBigRepoR(TestBase): - - """TestCase providing access to readonly 'big' repositories using the following - member variables: - - * gitrepopath - - * read-only base path of the git source repository, i.e. .../git/.git""" - - def setUp(self): - try: - super(TestBigRepoR, self).setUp() - except AttributeError: - pass - - self.gitrepopath = os.environ.get(k_env_git_repo) - if not self.gitrepopath: - logging.info( - "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", k_env_git_repo) - ospd = os.path.dirname - self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') - # end assure gitrepo is set - assert self.gitrepopath.endswith('.git') + """A placeholder in case we want to add additional functionality to all performance test-cases + """ #} END base classes diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index ed0a885bf..6e80bf5c6 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,10 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from gitdb.test.lib import ( - TestBase, - fixture_path -) +import os +from gitdb.test.lib import TestBase from gitdb import IStream from gitdb.db import LooseObjectDB @@ -16,7 +14,7 @@ class TestExamples(TestBase): def test_base(self): - ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + ldb = LooseObjectDB(os.path.join(self.gitrepopath, 'objects')) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) From f2233fbf40f3f69309ce5cc714e99fcbdcd33ec3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jan 2015 17:49:05 +0100 Subject: [PATCH 0301/1790] Removed unused imports - should have been in the last commit obviously --- gitdb/test/db/test_git.py | 1 - gitdb/test/db/test_ref.py | 1 - 2 files changed, 2 deletions(-) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index f28ffb761..2bda18f84 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -5,7 +5,6 @@ import os from gitdb.test.db.lib import ( TestDBBase, - fixture_path, with_rw_directory ) from gitdb.exc import BadObject diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 25cf37d1b..0e90f938b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -5,7 +5,6 @@ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, - fixture_path ) from gitdb.db import ReferenceDB From a88a777df3909a61be97f1a7b1194dad6de25702 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jan 2015 18:25:46 +0100 Subject: [PATCH 0302/1790] Make tests independent of actual repository data Therefore, hardcoded sha's are not allowed anymore, as the contents of the repository is unknown. Fixes #16, for real this time ;) --- gitdb/test/db/test_git.py | 7 ++++--- gitdb/test/db/test_ref.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 2bda18f84..acc0f153f 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -10,7 +10,7 @@ from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo -from gitdb.util import hex_to_bin, bin_to_hex +from gitdb.util import bin_to_hex class TestGitDB(TestDBBase): @@ -22,7 +22,7 @@ def test_reading(self): assert 1 < len(gdb.databases()) < 4 # access should be possible - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + gitdb_sha = next(gdb.sha_iter()) assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) ni = 50 @@ -35,7 +35,8 @@ def test_reading(self): # have a separate test module # test partial shas # this one as uneven and quite short - assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + gitdb_sha_hex = bin_to_hex(gitdb_sha) + assert gdb.partial_to_complete_sha_hex(gitdb_sha_hex[:5]) == gitdb_sha # mix even/uneven hexshas for i, binsha in enumerate(sha_list): diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 0e90f938b..6bac245c1 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -45,7 +45,7 @@ def test_writing(self, path): assert len(rdb.databases()) == 1 # we should now find a default revision of ours - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + gitdb_sha = next(rdb.sha_iter()) assert rdb.has_object(gitdb_sha) # remove valid From 13ad9b1199331a35e23f65c735acf482be09eae3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 8 Jan 2015 13:10:28 -0500 Subject: [PATCH 0303/1790] minor spell fixes + empty line unification + comparison for python 2.6 --- gitdb/base.py | 6 +++--- gitdb/exc.py | 2 -- gitdb/stream.py | 6 +++--- gitdb/util.py | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/gitdb/base.py b/gitdb/base.py index 5760b8af0..42e71d0fa 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -19,7 +19,7 @@ class OInfo(tuple): - """Carries information about an object in an ODB, provding information + """Carries information about an object in an ODB, providing information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. @@ -29,7 +29,7 @@ class OInfo(tuple): assert dbi[1] == dbi.type assert dbi[2] == dbi.size - The type is designed to be as lighteight as possible.""" + The type is designed to be as lightweight as possible.""" __slots__ = tuple() def __new__(cls, sha, type, size): @@ -69,7 +69,7 @@ class OPackInfo(tuple): does not include a sha. Additionally, the pack_offset is the absolute offset into the packfile at which - all object information is located. The data_offset property points to the abosolute + all object information is located. The data_offset property points to the absolute location in the pack at which that actual data stream can be found.""" __slots__ = tuple() diff --git a/gitdb/exc.py b/gitdb/exc.py index 817ac7b62..947e5d8bf 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -7,7 +7,6 @@ class ODBError(Exception): - """All errors thrown by the object database""" @@ -44,5 +43,4 @@ class BadObjectType(ODBError): class UnsupportedOperation(ODBError): - """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/stream.py b/gitdb/stream.py index 826def387..aaf5820bf 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -62,7 +62,7 @@ class DecompressMemMapReader(LazyMixin): hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap - and decompress it into chunks, thats all ... """ + and decompress it into chunks, that's all ... """ __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', '_cbr', '_phi') @@ -128,7 +128,7 @@ def new(self, m, close_on_deletion=False): This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. - :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param m: memory map on which to operate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) @@ -175,7 +175,7 @@ def compressed_bytes_read(self): # Only scrub the stream forward if we are officially done with the # bytes we were to have. if self._br == self._s and not self._zip.unused_data: - # manipulate the bytes-read to allow our own read method to coninute + # manipulate the bytes-read to allow our own read method to continue # but keep the window at its current position self._br = 0 if hasattr(self._zip, 'status'): diff --git a/gitdb/util.py b/gitdb/util.py index 8f80156b2..242be4405 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -18,7 +18,7 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -if sys.version_info[1] < 6: +if sys.version_info < (2, 6): mman = StaticWindowMapManager() else: mman = SlidingWindowMapManager() From 5b0dc5f89a666f450f39ef0002cf6d1761ecfca8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 12 Jan 2015 19:10:42 +0100 Subject: [PATCH 0304/1790] Adjusted stream logic to make it work on all tested platforms ... . As taken from https://github.com/gitpython-developers/gitdb/blob/master/gitdb/stream.py#L292 -> NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. They are thorough, and I assume it is truly working. Why is this logic as convoluted as it is ? Please look at the table in https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch will be the one that works. However, the zlib VERSIONs as well as the platform check is used to further match the entries in the table in the github issue. This is it ... it was the only way I could make this work everywhere. IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . <- Fixes #19 --- gitdb/fun.py | 2 +- gitdb/pack.py | 1 + gitdb/stream.py | 12 +++++++++--- setup.py | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 17da4e5da..ac9d99395 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -426,7 +426,7 @@ def pack_object_header_info(data): s = 4 # starting bit-shift size if PY3: while c & 0x80: - c = data[i] + c = byte_ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 diff --git a/gitdb/pack.py b/gitdb/pack.py index d2666d601..511e5571c 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -552,6 +552,7 @@ def _iter_objects(self, start_offset, as_stream=True): # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) + assert ostream.stream._br == ostream.size cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand diff --git a/gitdb/stream.py b/gitdb/stream.py index aaf5820bf..04dd79f90 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -289,11 +289,18 @@ def read(self, size=-1): # if we hit the end of the stream # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. # They are thorough, and I assume it is truly working. - if PY26: + # Why is this logic as convoluted as it is ? Please look at the table in + # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. + # Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # table in the github issue. This is it ... it was the only way I could make this work everywhere. + # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . + if PY26 or ((zlib.ZLIB_VERSION == '1.2.7' or zlib.ZLIB_VERSION == '1.2.5') and not sys.platform == 'darwin'): unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) - # end handle very special case ... + # # end handle very special case ... self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) @@ -374,7 +381,6 @@ def _set_cache_too_slow_without_c(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) # call len directly, as the (optional) c version doesn't implement the sequence diff --git a/setup.py b/setup.py index e634e37f2..be2f6e211 100755 --- a/setup.py +++ b/setup.py @@ -118,6 +118,7 @@ def get_data_files(self): "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", ],) From b3237e804ae313503f5479349f90066c356b1548 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 12 Jan 2015 20:38:38 +0100 Subject: [PATCH 0305/1790] Bumped version to 0.6.4 --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2ba8725eb..6554cf904 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 3) +version_info = (0, 6, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index be2f6e211..e4b1b1db1 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 3) +version_info = (0, 6, 4) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext': build_ext_nofail}, From 9aae93ea584c8cf9d1539a60e41c5c37119401d6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Jan 2015 18:16:34 +0100 Subject: [PATCH 0306/1790] Added issuestats to readme file --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 186218d1f..766d9d66a 100644 --- a/README.rst +++ b/README.rst @@ -49,6 +49,12 @@ DEVELOPMENT .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb +.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr + :target: http://www.issuestats.com/github/gitpython-developers/gitdb + +.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue + :target: http://www.issuestats.com/github/gitpython-developers/gitdb + The library is considered mature, and not under active development. It's primary (known) use is in git-python. INFRASTRUCTURE From a8af40edd969d79f1059fef774f5c7d600571999 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Jan 2015 18:18:18 +0100 Subject: [PATCH 0307/1790] Added issuestats badges to readme file --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e15c9bfc5..84b84afd6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Although memory maps have many advantages, they represent a very limited system [![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) [![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) +[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) +[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. From 5a3877a091d16a064a6ec07b1e41e536580831bb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Jan 2015 18:20:30 +0100 Subject: [PATCH 0308/1790] Fixed urls, they changed after moving the repo to gitpython-developers --- README.md | 8 ++++---- doc/source/intro.rst | 4 ++-- setup.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 84b84afd6..1c09ecb89 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ Although memory maps have many advantages, they represent a very limited system ## Overview -[![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) -[![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) +[![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) +[![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) @@ -63,7 +63,7 @@ It is advised to have a look at the **Usage Guide** for a brief introduction on ## Homepage and Links -The project is home on github at https://github.com/Byron/smmap . +The project is home on github at https://github.com/gitpython-developers/smmap . The latest source can be cloned from github as well: @@ -77,7 +77,7 @@ For support, please use the git-python mailing list: Issues can be filed on github: -* https://github.com/Byron/smmap/issues +* https://github.com/gitpython-developers/smmap/issues ## License Information diff --git a/doc/source/intro.rst b/doc/source/intro.rst index ee3108a6a..15f5bf08a 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -51,7 +51,7 @@ It is advised to have a look at the :ref:`Usage Guide ` for a br ################## Homepage and Links ################## -The project is home on github at `https://github.com/Byron/smmap `_. +The project is home on github at `https://github.com/gitpython-developers/smmap `_. The latest source can be cloned from github as well: @@ -65,7 +65,7 @@ For support, please use the git-python mailing list: Issues can be filed on github: - * https://github.com/Byron/smmap/issues + * https://github.com/gitpython-developers/smmap/issues ################### License Information diff --git a/setup.py b/setup.py index c6afc8960..267dfc7a5 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ if os.path.exists("README.md"): long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/Byron/smmap" + long_description = "See http://github.com/gitpython-developers/smmap" setup( name="smmap", From 016b58f3a7638d59ee766433649253e2d53e18b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20M=2E=20Bravo?= Date: Tue, 7 Apr 2015 08:32:38 -0500 Subject: [PATCH 0309/1790] Duplicate `const` fixed Remove duplicate `const` to stop the warning: "duplicate 'const' declaration specifier" --- gitdb/_delta_apply.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index 8b0f8e064..f4fffdcce 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -413,7 +413,7 @@ uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) // return size of the given delta info item inline -uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo const* veclast) +uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo* const veclast) { if (veclast == di){ return vec->di_last_size; @@ -534,7 +534,7 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) } } - const DeltaInfo const* vecend = DIV_end(src); + const DeltaInfo* const vecend = DIV_end(src); const uchar* nstream; for( ;cdi < vecend; ++cdi){ nstream = next_delta_info(src->dstream + cdi->dso, &dc); @@ -753,7 +753,7 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) PyObject* tmpargs = PyTuple_New(1); const uchar* data = TSI_first(&self->istream); - const uchar const* dend = TSI_end(&self->istream); + const uchar* const dend = TSI_end(&self->istream); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -979,8 +979,8 @@ PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar const* dstart = data; - const uchar const* dend = data + dlen; + const uchar* const dstart = data; + const uchar* const dend = data + dlen; div.dstream = dstart; if (dlen > pow(2, 32)){ From 18e4aea23644ea43657cb2e6846b6aaf78720c27 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 May 2015 08:56:47 +0200 Subject: [PATCH 0310/1790] fix(tests): remove line failing on power-pc It's worth noting that I never reproduced the issue, nor have I seen a stack-trace. Thus the line is removed in good-faith, but should also not pose any problem considering it was very specific and only in a test-case. The main problem is that I don't understand anymore why that assertion should be true, and thus can't judge the correctness of this fix at all. Closes #25 --- smmap/test/test_util.py | 8 -------- smmap/util.py | 1 - 2 files changed, 9 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 0a162607e..5a9d7bdf2 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -76,14 +76,6 @@ def test_region(self): assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) - # with the values we have, this test only works on windows where an alignment - # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions - # as they use different mapping techniques to circumvent the missing offset - # argument of mmap. - if sys.platform != 'win32': - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - # END handle platforms # auto-refcount assert rfull.client_count() == 1 diff --git a/smmap/util.py b/smmap/util.py index 1d20e5040..36ff1ab89 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -1,7 +1,6 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" import os import sys -import mmap from mmap import mmap, ACCESS_READ try: From 17413029b0f780ac94c24ab2b5f527ded6abdd2a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 22 Aug 2015 16:39:34 +0200 Subject: [PATCH 0311/1790] docs(gitdb): discourage usage of GitDB type --- gitdb/db/git.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gitdb/db/git.py b/gitdb/db/git.py index a4f6f54c3..7a43d7235 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -22,7 +22,11 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' - subdirectory""" + subdirectory + + ``IMPORTANT``: The usage of this implementation is highly discouraged as it fails to release file-handles. + This can be a problem with long-running processes and/or big repositories. + """ # Configuration PackDBCls = PackedDB LooseDBCls = LooseObjectDB From 2389b75280efb1a63e6ea578eae7f897fd4beb1b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 Oct 2015 19:17:44 +0200 Subject: [PATCH 0312/1790] fix(loose): avoid unnecessary file rename on windows This should workaround possible permission issues. Related to https://github.com/gitpython-developers/GitPython/issues/353 --- gitdb/db/loose.py | 13 +++++++++---- gitdb/ext/smmap | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4732b56da..1355e1cee 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -226,10 +226,15 @@ def store(self, istream): mkdir(obj_dir) # END handle destination directory # rename onto existing doesn't work on windows - if os.name == 'nt' and isfile(obj_path): - remove(obj_path) - # END handle win322 - rename(tmp_path, obj_path) + if os.name == 'nt': + if isfile(obj_path): + remove(tmp_path) + else: + rename(tmp_path, obj_path) + # end rename only if needed + else: + rename(tmp_path, obj_path) + # END handle win32 # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 84929ed81..18e4aea23 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 84929ed811142e366d6c5916125302c1419acad6 +Subproject commit 18e4aea23644ea43657cb2e6846b6aaf78720c27 From 7a8b138290ccf75da9ec5812dc7b55a40e8eeaeb Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Thu, 15 Oct 2015 09:57:26 +0800 Subject: [PATCH 0313/1790] Add Python 3.5 to test with travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b967f502e..464fafb2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 2.7 - 3.3 - 3.4 + - 3.5 install: - pip install coveralls script: From 50d073bd7cf769686efc63e42cb75512952b25ed Mon Sep 17 00:00:00 2001 From: Jesse Weigert Date: Tue, 27 Oct 2015 14:02:20 -0700 Subject: [PATCH 0314/1790] Fix package description --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 267dfc7a5..f6a04827d 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name="smmap", version=smmap.__version__, - description="A pure git implementation of a sliding window memory map manager", + description="A pure python implementation of a sliding window memory map manager", author=smmap.__author__, author_email=smmap.__contact__, url=smmap.__homepage__, From 490fdc2f27ca91898f09defdf20e1237a5ac12aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 30 Nov 2015 08:50:56 +0100 Subject: [PATCH 0315/1790] fix(distribution): remove redundant self-reference --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8a4cd3979..ed4898ec2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -gitdb -smmap>=0.8.3 \ No newline at end of file +smmap>=0.8.3 From d1996e04dbf4841b853b60c1365f0f5fd28d170c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 28 Mar 2016 09:10:31 +0200 Subject: [PATCH 0316/1790] Ignore MANIFEST.in Fixes #25 --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 597944fd4..b939b5ded 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE include CHANGES include AUTHORS include README +include MANIFEST.in include gitdb/_fun.c include gitdb/_delta_apply.c From 5ff376161a2f2875d3fc0eb1d9f25027e29460ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 08:49:52 +0300 Subject: [PATCH 0317/1790] travis: Test with Python 3.5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8cab8225f..ba0beaa7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "2.7" - "3.3" - "3.4" + - "3.5" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) git: From cadb3d8a095c3e51cede1c33f7dcbf49c1426418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 08:53:11 +0300 Subject: [PATCH 0318/1790] setup: Add Python 3.5 classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e4b1b1db1..c2f9f1eb1 100755 --- a/setup.py +++ b/setup.py @@ -121,4 +121,5 @@ def get_data_files(self): "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", ],) From 2af455266cb3ea454c4f38b182825b9e78b8398d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 09:32:29 +0300 Subject: [PATCH 0319/1790] Spelling fixes --- doc/source/algorithm.rst | 2 +- doc/source/changes.rst | 2 +- gitdb/db/base.py | 2 +- gitdb/db/loose.py | 2 +- gitdb/db/mem.py | 2 +- gitdb/pack.py | 4 ++-- gitdb/stream.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/lib.py | 2 +- gitdb/test/test_pack.py | 2 +- gitdb/test/test_util.py | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index 4374cb820..2e01b3fbe 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -92,6 +92,6 @@ Future work Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. -The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be paid with extra pre-processing time. A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index a36fd659c..22deb6db3 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -8,7 +8,7 @@ Changelog * Fixed possibly critical error, see https://github.com/gitpython-developers/GitPython/issues/220 - - However, it only seems to occour on high-entropy data and didn't reoccour after the fix + - However, it only seems to occur on high-entropy data and didn't reoccour after the fix ***** 0.6.0 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2615b13cc..2d7b9fa8d 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -177,7 +177,7 @@ def _db_query(self, sha): """:return: database containing the given 20 byte sha :raise BadObject:""" # most databases use binary representations, prevent converting - # it everytime a database is being queried + # it every time a database is being queried try: return self._db_cache[sha] except KeyError: diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 1355e1cee..192c524af 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -174,7 +174,7 @@ def has_object(self, sha): return True except BadObject: return False - # END check existance + # END check existence def store(self, istream): """note: The sha we produce will be hex by nature""" diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 595dbf4fc..871133468 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -98,7 +98,7 @@ def stream_copy(self, sha_iter, odb): for sha in sha_iter: if odb.has_object(sha): continue - # END check object existance + # END check object existence ostream = self.stream(sha) # compressed data including header diff --git a/gitdb/pack.py b/gitdb/pack.py index 511e5571c..2447455b5 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -85,7 +85,7 @@ def pack_object_at(cursor, offset, as_stream): an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. - :param data: random accessable data containing all required information + :param data: random accessible data containing all required information :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" @@ -447,7 +447,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): :return: index as in `sha_to_index` or None if the sha was not found in this index file :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes - :param canonical_length: lenght of the original hexadecimal representation of the + :param canonical_length: length of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: diff --git a/gitdb/stream.py b/gitdb/stream.py index 04dd79f90..be95c113a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -660,7 +660,7 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" + :return: length of incoming data""" self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 024c194a2..9c25a0249 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -33,4 +33,4 @@ def test_basics(self, path): # END for each sha self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') - # raises if no object could be foudn + # raises if no object could be found diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index a089eac6f..bbdd241cd 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -76,7 +76,7 @@ def wrapper(self, *args, **kwargs): def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the - test suceeds, but leave it otherwise to aid additional debugging""" + test succeeds, but leave it otherwise to aid additional debugging""" def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index ff1057237..a8b0b60a4 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -232,7 +232,7 @@ def rewind_streams(): # END verify files exist # END for each packpath, indexpath pair - # verify the packs throughly + # verify the packs thoroughly rewind_streams() entity = PackEntity.create(pack_objs, rw_dir) count = 0 diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 1dee54461..5026f4e62 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -60,7 +60,7 @@ def test_lockedfd(self): self._cmp_contents(my_file, orig_data) assert not os.path.isfile(lockfilepath) - # additional call doesnt fail + # additional call doesn't fail lfd.commit() lfd.rollback() From f957c812ac3221773058ba2fa8cd38017537da8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 09:34:09 +0300 Subject: [PATCH 0320/1790] Handle more file open/close with "with" --- gitdb/db/ref.py | 3 ++- gitdb/test/db/test_ref.py | 7 +++---- gitdb/test/test_pack.py | 5 ++--- gitdb/test/test_util.py | 10 +++------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 83a9f611d..2e3db86db 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -41,7 +41,8 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + with open(self._ref_file, 'r') as f: + ref_paths = [l.strip() for l in f] except (OSError, IOError): pass # END handle alternates diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 6bac245c1..20496985e 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -21,10 +21,9 @@ class TestReferenceDB(TestDBBase): def make_alt_file(self, alt_path, alt_list): """Create an alternates file which contains the given alternates. The list can be empty""" - alt_file = open(alt_path, "wb") - for alt in alt_list: - alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) - alt_file.close() + with open(alt_path, "wb") as alt_file: + for alt in alt_list: + alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index ff1057237..8ecfcd91a 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -197,7 +197,6 @@ def rewind_streams(): obj.stream.seek(0) # END utility for ppath, ipath, num_obj in zip((pack_path, ) * 2, (index_path, None), (len(pack_objs), None)): - pfile = open(ppath, 'wb') iwrite = None if ipath: ifile = open(ipath, 'wb') @@ -210,8 +209,8 @@ def rewind_streams(): # END rewind streams iteration += 1 - pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) - pfile.close() + with open(ppath, 'wb') as pfile: + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) assert os.path.getsize(ppath) > 100 # verify pack diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 1dee54461..c2989203e 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -25,19 +25,15 @@ def test_basics(self): def _cmp_contents(self, file_path, data): # raise if data from file at file_path # does not match data string - fp = open(file_path, "rb") - try: + with open(file_path, "rb") as fp: assert fp.read() == data.encode("ascii") - finally: - fp.close() def test_lockedfd(self): my_file = tempfile.mktemp() orig_data = "hello" new_data = "world" - my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data.encode("ascii")) - my_file_fp.close() + with open(my_file, "wb") as my_file_fp: + my_file_fp.write(orig_data.encode("ascii")) try: lfd = LockedFD(my_file) From 36ed59a1bb44ecada33ce83049605fd7d70e7876 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 7 Sep 2016 14:59:29 +0100 Subject: [PATCH 0321/1790] Support universal wheels --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..3c6e79cf3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 From fde2dde8e6fe39c8548acb0c919cf18adaf2806a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 12 Sep 2016 20:16:22 +0100 Subject: [PATCH 0322/1790] Do not support universal wheels because you need to build manylinux1/mac/windows wheels instead --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3c6e79cf3..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 From e835f555c2d78601228c73ef048826c1f108cec7 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 21:45:21 +0200 Subject: [PATCH 0323/1790] ci: Enable Appveyor. --- .appveyor.yml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..2daadaa4d --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,49 @@ +# CI on Windows via appveyor +environment: + + matrix: + ## MINGW + # + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + - PYTHON: "C:\\Miniconda35-x64" + PYTHON_VERSION: "3.5" + IS_CONDA: "yes" + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + + ## Print configuration for debugging. + # + - | + echo %PATH% + uname -a + where python pip pip2 pip3 pip34 + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + + - IF "%IS_CONDA%"=="yes" ( + conda info -a & + conda install --yes --quiet pip + ) + - pip install nose wheel coveralls + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - pip install -e . + +build: false + +test_script: + - IF "%PYTHON_VERSION%"=="3.5" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) From dfadc166beaf28ba216e03e3592649437397fadb Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 22:23:46 +0200 Subject: [PATCH 0324/1790] Appveyor: Add badge. --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c09ecb89..ef0b23601 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview [![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) +[![Build status](https://ci.appveyor.com/api/projects/status/h8rl7thsr42oc0pf?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) [![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) @@ -44,15 +45,15 @@ The package was tested on all of the previously mentioned configurations. [![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: - + ```bash $ pip install smmap ``` - + As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. If you have downloaded the source archive, the package can be installed by running the `setup.py` script: - + ```bash $ python setup.py install ``` @@ -68,17 +69,17 @@ The project is home on github at https://github.com/gitpython-developers/smmap . The latest source can be cloned from github as well: * git://github.com/gitpython-developers/smmap.git - - + + For support, please use the git-python mailing list: * http://groups.google.com/group/git-python - + Issues can be filed on github: * https://github.com/gitpython-developers/smmap/issues - + ## License Information From be5d4267f6cc4272e3fbfa1fbd262e984c2077bf Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 2 Oct 2016 01:28:06 +0200 Subject: [PATCH 0325/1790] io: retrofit classes wih destructors into context-mans --- smmap/buf.py | 12 +- smmap/mman.py | 6 + smmap/test/lib.py | 13 +- smmap/test/test_buf.py | 202 +++++++++++------------ smmap/test/test_mman.py | 317 ++++++++++++++++++------------------ smmap/test/test_tutorial.py | 121 +++++++------- smmap/test/test_util.py | 48 +++--- 7 files changed, 368 insertions(+), 351 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index e6f24341d..438292b60 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -47,6 +47,12 @@ def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): def __del__(self): self.end_access() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end_access() + def __len__(self): return self._size @@ -83,7 +89,7 @@ def __getslice__(self, i, j): # in the previous iteration of this code pyvers = sys.version_info[:2] if (3, 0) <= pyvers <= (3, 3): - # Memory view cannot be joined below python 3.4 ... + # Memory view cannot be joined below python 3.4 ... out = bytes() while l: c.use_region(ofs, l) @@ -91,7 +97,7 @@ def __getslice__(self, i, j): d = c.buffer()[:l] ofs += len(d) l -= len(d) - # This is slower than the join ... but what can we do ... + # This is slower than the join ... but what can we do ... out += d del(d) # END while there are bytes to read @@ -104,7 +110,7 @@ def __getslice__(self, i, j): d = c.buffer()[:l] ofs += len(d) l -= len(d) - # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but # can't unless we use pure bytes if hasattr(d, 'tobytes'): d = d.tobytes() diff --git a/smmap/mman.py b/smmap/mman.py index 7180c75b3..af12ee98f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -46,6 +46,12 @@ def __init__(self, manager=None, regions=None): def __del__(self): self._destroy() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._destroy() + def _destroy(self): """Destruction code to decrement counters""" self.unuse_region() diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 93cb09a5e..f86c0c6f1 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -21,10 +21,9 @@ def __init__(self, size, prefix=''): self._path = tempfile.mktemp(prefix=prefix) self._size = size - fp = open(self._path, "wb") - fp.seek(size - 1) - fp.write(b'1') - fp.close() + with open(self._path, "wb") as fp: + fp.seek(size - 1) + fp.write(b'1') assert os.path.getsize(self.path) == size @@ -35,6 +34,12 @@ def __del__(self): pass # END exception handling + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.__del__() + @property def path(self): return self._path diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 984b43254..3b6009e11 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -25,104 +25,104 @@ class TestBuf(TestBase): def test_basics(self): - fc = FileCreator(self.k_window_test_size, "buffer_test") - - # invalid paths fail upon construction - c = man_optimal.make_cursor(fc.path) - self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - - buf = SlidingWindowMapBuffer() # can create uninitailized buffers - assert buf.cursor() is None - - # can call end access any time - buf.end_access() - buf.end_access() - assert len(buf) == 0 - - # begin access can revive it, if the offset is suitable - offset = 100 - assert buf.begin_access(c, fc.size) == False - assert buf.begin_access(c, offset) == True - assert len(buf) == fc.size - offset - assert buf.cursor().is_valid() - - # empty begin access keeps it valid on the same path, but alters the offset - assert buf.begin_access() == True - assert len(buf) == fc.size - assert buf.cursor().is_valid() - - # simple access - with open(fc.path, 'rb') as fp: - data = fp.read() - assert data[offset] == buf[0] - assert data[offset:offset * 2] == buf[0:offset] - - # negative indices, partial slices - assert buf[-1] == buf[len(buf) - 1] - assert buf[-10:] == buf[len(buf) - 10:len(buf)] - - # end access makes its cursor invalid - buf.end_access() - assert not buf.cursor().is_valid() - assert buf.cursor().is_associated() # but it remains associated - - # an empty begin access fixes it up again - assert buf.begin_access() == True and buf.cursor().is_valid() - del(buf) # ends access automatically - del(c) - - assert man_optimal.num_file_handles() == 1 - - # PERFORMANCE - # blast away with random access and a full mapping - we don't want to - # exaggerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which - # will produce small mappings only ! - max_num_accesses = 100 - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - for manager, man_id in ((man_optimal, 'optimal'), - (man_worst_case, 'worst case'), - (static_man, 'static optimal')): - buf = SlidingWindowMapBuffer(manager.make_cursor(item)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - del d - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - # END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000 * 1000) - mode_str = (access_mode and "slice") or "single byte" - print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), - file=sys.stderr) - # END handle access mode - del buf - # END for each manager - # END for each input - os.close(fd) + with FileCreator(self.k_window_test_size, "buffer_test") as fc: + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitailized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + with open(fc.path, 'rb') as fp: + data = fp.read() + assert data[offset] == buf[0] + assert data[offset:offset * 2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ((man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimal')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + del d + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + # END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000 * 1000) + mode_str = (access_mode and "slice") or "single byte" + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), + file=sys.stderr) + # END handle access mode + del buf + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index c8b9c703e..96bc355b7 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -19,19 +19,18 @@ class TestMMan(TestBase): def test_cursor(self): - fc = FileCreator(self.k_window_test_size, "cursor_test") - - man = SlidingWindowMapManager() - ci = WindowCursor(man) # invalid cursor - assert not ci.is_valid() - assert not ci.is_associated() - assert ci.size() == 0 # this is cached, so we can query it in invalid state - - cv = man.make_cursor(fc.path) - assert not cv.is_valid() # no region mapped yet - assert cv.is_associated() # but it know where to map it from - assert cv.file_size() == fc.size - assert cv.path() == fc.path + with FileCreator(self.k_window_test_size, "cursor_test") as fc: + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated() # but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path # copy module cio = copy(cv) @@ -74,150 +73,154 @@ def test_memory_manager(self): assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - with open(fc.path, 'rb') as fp: - assert c.buffer()[:] == fp.read(20)[10:] - - if isinstance(item, int): - self.assertRaises(ValueError, c.path) - else: - self.assertRaises(ValueError, c.fd) - # END handle value error - # END for each input - os.close(fd) - # END for each manager type + with FileCreator(self.k_window_test_size, "manager_test") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + + if isinstance(item, int): + self.assertRaises(ValueError, c.path) + else: + self.assertRaises(ValueError, c.fd) + # END handle value error + # END for each input + finally: + os.close(fd) + # END for each manasger type def test_memman_operation(self): # test more access, force it to actually unmap regions - fc = FileCreator(self.k_window_test_size, "manager_operation_test") - with open(fc.path, 'rb') as fp: - data = fp.read() - fd = os.open(fc.path, os.O_RDONLY) - max_num_handles = 15 - # small_size = - for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() // 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region() - assert rr.client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr.size() - - # assert c.size() == size # the cursor may overallocate in its static version - assert c.ofs_begin() == base_offset - assert rr.ofs_begin() == 0 # it was aligned and expanded - if man.window_size(): - # but isn't larger than the max window (aligned) - assert rr.size() == align_to_mmap(man.window_size(), True) - else: - assert rr.size() == fc.size - # END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] - - # obtain second window, which spans the first part of the file - it is a still the same window - nsize = (size or fc.size) - 10 - assert c.use_region(0, nsize).is_valid() - assert c.region() == rr - assert man.num_file_handles() == 1 - assert c.size() == nsize - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:nsize] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - (size or c.size()) + overshoot - assert c.use_region(base_offset, size).is_valid() - if man.window_size(): - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region() is not rr # old region is still available, but has not curser ref anymore - assert rr.client_count() == 1 # only held by manager - else: - assert c.size() < fc.size - # END ignore static managers which only have one handle per file - rr = c.region() - assert rr.client_count() == 2 # manager + cursor - assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr.ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - # END ignore this for static managers - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) - - # precondition - if man.window_size(): - assert max_mapped_memory_size >= mapped_memory_size() - # END statics will overshoot, which is fine - assert max_file_handles >= num_file_handles() - assert c.use_region(base_offset, (size or c.size())).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset + csize] - memory_read += csize - - assert includes_ofs(base_offset) - assert includes_ofs(base_offset + csize - 1) - assert not includes_ofs(base_offset + csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), - file=sys.stderr) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - # END for each item - # END for each manager type - os.close(fd) + with FileCreator(self.k_window_test_size, "manager_operation_test") as fc: + with open(fc.path, 'rb') as fp: + data = fp.read() + fd = os.open(fc.path, os.O_RDONLY) + try: + max_num_handles = 15 + # small_size = + for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() // 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region() + assert rr.client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr.size() + + # assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr.ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + # but isn't larger than the max window (aligned) + assert rr.size() == align_to_mmap(man.window_size(), True) + else: + assert rr.size() == fc.size + # END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region() == rr + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region() is not rr # old region is still available, but has not curser ref anymore + assert rr.client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + # END ignore static managers which only have one handle per file + rr = c.region() + assert rr.client_count() == 2 # manager + cursor + assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr.ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + # END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + # END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset + csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), + file=sys.stderr) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + # END for each item + # END for each manager type + finally: + os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index f7a2128a9..b03db9be2 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -20,65 +20,62 @@ def test_example(self): # Cursors ########## import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") - - # obtain a cursor to access some file. - c = mman.make_cursor(fc.path) - - # the cursor is now associated with the file, but not yet usable - assert c.is_associated() - assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to - # access. The following just says you want as much data as possible starting - # from offset 0. - # To be sure your region could be mapped, query for validity - assert c.use_region().is_valid() # use_region returns self - - # once a region was mapped, you must query its dimension regularly - # to assure you don't try to access its buffer out of its bounds - assert c.size() - c.buffer()[0] # first byte - c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size() - 1] # last byte - - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - from smmap.util import buffer - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - - # you can query absolute offsets, and check whether an offset is included - # in the cursor's data. - assert c.ofs_begin() < c.ofs_end() - assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the - # cursor will be come invalid. It cannot be used in that state - assert not c.use_region(fc.size, 100).is_valid() - # map as much as possible after skipping the first 100 bytes - assert c.use_region(100).is_valid() - - # You can explicitly free cursor resources by unusing the cursor's region - c.unuse_region() - assert not c.is_valid() - - # Buffers - ######### - # Create a default buffer which can operate on the whole file - buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - - # you can use it right away - assert buf.cursor().is_valid() - - buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:] # access the last ten bytes - - # If you want to keep the instance between different accesses, use the - # dedicated methods - buf.end_access() - assert not buf.cursor().is_valid() # you cannot use the buffer anymore - assert buf.begin_access(offset=10) # start using the buffer at an offset - - # it will stop using resources automatically once it goes out of scope + with smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") as fc: + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size() - 1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + from smmap.util import buffer + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 5a9d7bdf2..e6ac10f35 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -60,22 +60,22 @@ def test_window(self): assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) def test_region(self): - fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size // 2 - rofs = align_to_mmap(4200, False) - rfull = MapRegion(fc.path, 0, fc.size) - rhalfofs = MapRegion(fc.path, rofs, fc.size) - rhalfsize = MapRegion(fc.path, 0, half_size) + with FileCreator(self.k_window_test_size, "window_test") as fc: + half_size = fc.size // 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # auto-refcount assert rfull.client_count() == 1 @@ -87,17 +87,17 @@ def test_region(self): assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): - fc = FileCreator(100, "sample_file") - - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - ml = MapRegionList(item) - - assert len(ml) == 0 - assert ml.path_or_fd() == item - assert ml.file_size() == fc.size - # END handle input - os.close(fd) + with FileCreator(100, "sample_file") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + finally: + os.close(fd) def test_util(self): assert isinstance(is_64_bit(), bool) # just call it From 62815b5c5a4c39e9ace3d20ec0c593011201dbcf Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 4 Oct 2016 18:04:24 +0100 Subject: [PATCH 0326/1790] support optional gitdb_speedups --- README.rst | 7 + gitdb/_delta_apply.c | 1154 ------------------------------------------ gitdb/_delta_apply.h | 6 - gitdb/_fun.c | 107 ---- gitdb/fun.py | 2 +- gitdb/pack.py | 2 +- gitdb/stream.py | 2 +- setup.cfg | 2 + setup.py | 154 ++---- 9 files changed, 49 insertions(+), 1387 deletions(-) delete mode 100644 gitdb/_delta_apply.c delete mode 100644 gitdb/_delta_apply.h delete mode 100644 gitdb/_fun.c create mode 100644 setup.cfg diff --git a/README.rst b/README.rst index 766d9d66a..1b754d09b 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,13 @@ From `PyPI `_ pip install gitdb +SPEEDUPS +======== + +If you want to go up to 20% faster, you can install gitdb-speedups with: + + pip install gitdb-speedups + REQUIREMENTS ============ diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c deleted file mode 100644 index f4fffdcce..000000000 --- a/gitdb/_delta_apply.c +++ /dev/null @@ -1,1154 +0,0 @@ -#include <_delta_apply.h> -#include -#include -#include -#include -#include - - - -typedef unsigned long long ull; -typedef unsigned int uint; -typedef unsigned char uchar; -typedef unsigned short ushort; -typedef uchar bool; - -// Constants -const ull gDIV_grow_by = 100; - - - -// DELTA STREAM ACCESS -/////////////////////// -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - - -// TOP LEVEL STREAM INFO -///////////////////////////// -typedef struct { - const uchar *tds; // Toplevel delta stream - const uchar *cstart; // start of the chunks - Py_ssize_t tdslen; // size of tds in bytes - Py_ssize_t target_size; // size of the target buffer which can hold all data - uint num_chunks; // amount of chunks in the delta stream - PyObject *parent_object; -} ToplevelStreamInfo; - - -void TSI_init(ToplevelStreamInfo* info) -{ - info->tds = NULL; - info->cstart = NULL; - info->tdslen = 0; - info->num_chunks = 0; - info->target_size = 0; - info->parent_object = 0; -} - -void TSI_destroy(ToplevelStreamInfo* info) -{ -#ifdef DEBUG - fprintf(stderr, "TSI_destroy: %p\n", info); -#endif - - if (info->parent_object){ - Py_DECREF(info->parent_object); - info->parent_object = NULL; - } else if (info->tds){ - PyMem_Free((void*)info->tds); - } - info->tds = NULL; - info->cstart = NULL; - info->tdslen = 0; - info->num_chunks = 0; -} - -inline -const uchar* TSI_end(ToplevelStreamInfo* info) -{ - return info->tds + info->tdslen; -} - -inline -const uchar* TSI_first(ToplevelStreamInfo* info) -{ - return info->cstart; -} - -// set the stream, and initialize it -// initialize our set stream to point to the first chunk -// Fill in the header information, which is the base and target size -inline -void TSI_set_stream(ToplevelStreamInfo* info, const uchar* stream) -{ - info->tds = stream; - info->cstart = stream; - - assert(info->tds && info->tdslen); - - // init stream - const uchar* tdsend = TSI_end(info); - msb_size(&info->cstart, tdsend); // base size - info->target_size = msb_size(&info->cstart, tdsend); -} - - - -// duplicate the data currently owned by the parent object drop its refcount -// return 1 on success -bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) -{ - assert(info->parent_object); - - uchar* ptmp = PyMem_Malloc(info->tdslen); - if (!ptmp){ - return 0; - } - uint ofs = (uint)(info->cstart - info->tds); - memcpy((void*)ptmp, info->tds, info->tdslen); - - info->tds = ptmp; - info->cstart = ptmp + ofs; - - Py_DECREF(info->parent_object); - info->parent_object = 0; - - return 1; -} - -// Transfer ownership of the given stream into our instance. The amount of chunks -// remains the same, and needs to be set by the caller -void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) -{ - assert(info->parent_object == 0); - - uint ofs = (uint)(info->cstart - info->tds); - if (info->tds){ - PyMem_Free((void*)info->tds); - } - info->tds = stream; - info->cstart = info->tds + ofs; - info->tdslen = streamlen; - -} - -// DELTA CHUNK -//////////////// -// Internal Delta Chunk Objects -// They are just used to keep information parsed from a stream -// The data pointer is always shared -typedef struct { - ull to; - uint ts; - uint so; - const uchar* data; -} DeltaChunk; - -// forward declarations -const uchar* next_delta_info(const uchar*, DeltaChunk*); - -inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) -{ - dc->to = to; - dc->ts = ts; - dc->so = so; - dc->data = NULL; -} - - -inline -ull DC_rbound(const DeltaChunk* dc) -{ - return dc->to + dc->ts; -} - -inline -void DC_print(const DeltaChunk* dc, const char* prefix) -{ - fprintf(stderr, "%s-dc: to = %i, ts = %i, so = %i, data = %p\n", prefix, (int)dc->to, dc->ts, dc->so, dc->data); -} - -// Apply -inline -void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) -{ - PyObject* buffer = 0; - if (dc->data){ - buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); - } else { - buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); - } - - if (PyTuple_SetItem(tmpargs, 0, buffer)){ - assert(0); - } - - - // tuple steals reference, and will take care about the deallocation - PyObject_Call(writer, tmpargs, NULL); - -} - -// Encode the information in the given delta chunk and write the byte-stream -// into the given output stream -// It will be copied into the given bounds, the given size must be the final size -// and work with the given relative offset - hence the bounds are assumed to be -// correct and to fit within the unaltered dc -inline -void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) -{ - uchar* out = *pout; - if (dc->data){ - *out++ = (uchar)size; - memcpy(out, dc->data+ofs, size); - out += size; - } else { - uchar i = 0x80; - uchar* op = out++; - uint moff = dc->so+ofs; - - if (moff & 0x000000ff) - *out++ = moff >> 0, i |= 0x01; - if (moff & 0x0000ff00) - *out++ = moff >> 8, i |= 0x02; - if (moff & 0x00ff0000) - *out++ = moff >> 16, i |= 0x04; - if (moff & 0xff000000) - *out++ = moff >> 24, i |= 0x08; - - if (size & 0x00ff) - *out++ = size >> 0, i |= 0x10; - if (size & 0xff00) - *out++ = size >> 8, i |= 0x20; - - *op = i; - } - - *pout = out; -} - -// Return: amount of bytes one would need to encode dc -inline -ushort DC_count_encode_bytes(const DeltaChunk* dc) -{ - if (dc->data){ - return 1 + dc->ts; // cmd byte + actual data bytes - } else { - ushort c = 1; // cmd byte - uint ts = dc->ts; - ull so = dc->so; - - // offset - c += (so & 0x000000FF) > 0; - c += (so & 0x0000FF00) > 0; - c += (so & 0x00FF0000) > 0; - c += (so & 0xFF000000) > 0; - - // size - max size is 0x10000, its encoded with 0 size bits - c += (ts & 0x000000FF) > 0; - c += (ts & 0x0000FF00) > 0; - - return c; - } -} - - - -// DELTA INFO -///////////// -typedef struct { - uint dso; // delta stream offset, relative to the very start of the stream - uint to; // target offset (cache) -} DeltaInfo; - - -// DELTA INFO VECTOR -////////////////////// - -typedef struct { - DeltaInfo *mem; // Memory for delta infos - uint di_last_size; // size of the last element - we can't compute it using the next bound - const uchar *dstream; // borrowed ointer to delta stream we index - Py_ssize_t size; // Amount of DeltaInfos - Py_ssize_t reserved_size; // Reserved amount of DeltaInfos -} DeltaInfoVector; - - - -// Reserve enough memory to hold the given amount of delta chunks -// Return 1 on success -// NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DIVs change often, and reallocs are expensive -inline -int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) -{ - if (num_dc <= vec->reserved_size){ - return 1; - } - -#ifdef DEBUG - bool was_null = vec->mem == NULL; -#endif - - if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaInfo)); - } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaInfo)); - } - - if (vec->mem == NULL){ - Py_FatalError("Could not allocate memory for append operation"); - } - - vec->reserved_size = num_dc; - -#ifdef DEBUG - const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; - if (!was_null) - format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaInfo)), vec->mem, (int)vec->reserved_size); -#endif - - return vec->mem != NULL; -} - -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ -inline -int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) -{ - return DIV_reserve_memory(vec, vec->reserved_size + num_dc); -} - -int DIV_init(DeltaInfoVector* vec, ull initial_size) -{ - vec->mem = NULL; - vec->dstream = NULL; - vec->size = 0; - vec->reserved_size = 0; - vec->di_last_size = 0; - - return DIV_grow_by(vec, initial_size); -} - -inline -Py_ssize_t DIV_len(const DeltaInfoVector* vec) -{ - return vec->size; -} - -inline -uint DIV_lbound(const DeltaInfoVector* vec) -{ - assert(vec->size && vec->mem); - return vec->mem->to; -} - -// Return item at index -inline -DeltaInfo* DIV_get(const DeltaInfoVector* vec, Py_ssize_t i) -{ - assert(i < vec->size && vec->mem); - return &vec->mem[i]; -} - -// Return last item -inline -DeltaInfo* DIV_last(const DeltaInfoVector* vec) -{ - return DIV_get(vec, vec->size-1); -} - -inline -int DIV_empty(const DeltaInfoVector* vec) -{ - return vec->size == 0; -} - -// Return end pointer of the vector -inline -const DeltaInfo* DIV_end(const DeltaInfoVector* vec) -{ - assert(!DIV_empty(vec)); - return vec->mem + vec->size; -} - -// return first item in vector -inline -DeltaInfo* DIV_first(const DeltaInfoVector* vec) -{ - assert(!DIV_empty(vec)); - return vec->mem; -} - -// return rbound offset in bytes. We use information contained in the -// vec to do that -inline -uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) -{ - if (DIV_last(vec) == di){ - return di->to + vec->di_last_size; - } else { - return (di+1)->to; - } -} - -// return size of the given delta info item -inline -uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo* const veclast) -{ - if (veclast == di){ - return vec->di_last_size; - } else { - return (di+1)->to - di->to; - } -} - -// return size of the given delta info item -inline -uint DIV_info_size(const DeltaInfoVector* vec, const DeltaInfo* di) -{ - return DIV_info_size2(vec, di, DIV_last(vec)); -} - -void DIV_destroy(DeltaInfoVector* vec) -{ - if (vec->mem){ -#ifdef DEBUG - fprintf(stderr, "DIV_destroy: %p\n", (void*)vec->mem); -#endif - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - -// Reset this vector so that its existing memory can be filled again. -// Memory will be kept, but not cleaned up -inline -void DIV_forget_members(DeltaInfoVector* vec) -{ - vec->size = 0; -} - -// Reset the vector so that its size will be zero -// It will keep its memory though, and hence can be filled again -inline -void DIV_reset(DeltaInfoVector* vec) -{ - if (vec->size == 0) - return; - vec->size = 0; -} - - -// Append one chunk to the end of the list, and return a pointer to it -// It will not have been initialized ! -inline -DeltaInfo* DIV_append(DeltaInfoVector* vec) -{ - if (vec->size + 1 > vec->reserved_size){ - DIV_grow_by(vec, gDIV_grow_by); - } - - DeltaInfo* next = vec->mem + vec->size; - vec->size += 1; - return next; -} - -// Return delta chunk being closest to the given absolute offset -inline -DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) -{ - assert(vec->mem); - - ull lo = 0; - ull hi = vec->size; - ull mid; - DeltaInfo* di; - - while (lo < hi) - { - mid = (lo + hi) / 2; - di = vec->mem + mid; - if (di->to > ofs){ - hi = mid; - } else if ((DIV_info_rbound(vec, di) > ofs) | (di->to == ofs)) { - return di; - } else { - lo = mid + 1; - } - } - - return DIV_last(vec); -} - - -// Return the amount of chunks a slice at the given spot would have, as well as -// its size in bytes it would have if the possibly partial chunks would be encoded -// and added to the spot marked by sdc -uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) -{ - uint num_bytes = 0; - DeltaInfo* cdi = DIV_closest_chunk(src, ofs); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - // partial overlap - if (cdi->to != ofs) { - const ull relofs = ofs - cdi->to; - const uint cdisize = DIV_info_size(src, cdi); - const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= max_size; - - // get the size in bytes the info would have - next_delta_info(src->dstream + cdi->dso, &dc); - dc.so += relofs; - dc.ts = max_size; - num_bytes += DC_count_encode_bytes(&dc); - - cdi += 1; - - if (size == 0){ - return num_bytes; - } - } - - const DeltaInfo* const vecend = DIV_end(src); - const uchar* nstream; - for( ;cdi < vecend; ++cdi){ - nstream = next_delta_info(src->dstream + cdi->dso, &dc); - - if (dc.ts < size) { - num_bytes += nstream - (src->dstream + cdi->dso); - size -= dc.ts; - } else { - dc.ts = size; - num_bytes += DC_count_encode_bytes(&dc); - size = 0; - break; - } - } - - assert(size == 0); - return num_bytes; -} - -// Write a slice as defined by its absolute offset in bytes and its size into the given -// destination memory. The individual chunks written will be a byte copy of the source -// data chunk stream -// Return: number of chunks in the slice -uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) -{ - assert(DIV_lbound(src) <= tofs); - assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - DeltaInfo* cdi = DIV_closest_chunk(src, tofs); - uint num_chunks = 0; - - // partial overlap - if (cdi->to != tofs) { - const uint relofs = tofs - cdi->to; - next_delta_info(src->dstream + cdi->dso, &dc); - const uint max_size = dc.ts - relofs < size ? dc.ts - relofs : size; - - size -= max_size; - - // adjust dc proportions - DC_encode_to(&dc, dest, relofs, max_size); - - num_chunks += 1; - cdi += 1; - - if (size == 0){ - return num_chunks; - } - } - - const uchar* dstream = src->dstream + cdi->dso; - const uchar* nstream = dstream; - for( ; nstream; dstream = nstream) - { - num_chunks += 1; - nstream = next_delta_info(dstream, &dc); - if (dc.ts < size) { - memcpy(*dest, dstream, nstream - dstream); - *dest += nstream - dstream; - size -= dc.ts; - } else { - DC_encode_to(&dc, dest, 0, size); - size = 0; - break; - } - } - - assert(size == 0); - return num_chunks; -} - - -// Take slices of div into the corresponding area of the tsi, which is the topmost -// delta to apply. -bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) -{ - assert(tsi->num_chunks); - - - uint num_bytes = 0; - const uchar* data = TSI_first(tsi); - const uchar* dend = TSI_end(tsi); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - - // COMPUTE SIZE OF TARGET STREAM - ///////////////////////////////// - for (;data < dend;) - { - data = next_delta_info(data, &dc); - - // Data chunks don't need processing - if (dc.data){ - num_bytes += 1 + dc.ts; - continue; - } - - num_bytes += DIV_count_slice_bytes(div, dc.so, dc.ts); - } - assert(DC_rbound(&dc) == tsi->target_size); - - - // GET NEW DELTA BUFFER - //////////////////////// - uchar *const dstream = PyMem_Malloc(num_bytes); - if (!dstream){ - return 0; - } - - - data = TSI_first(tsi); - const uchar *ndata = data; - dend = TSI_end(tsi); - - uint num_chunks = 0; - uchar* ds = dstream; - DC_init(&dc, 0, 0, 0, NULL); - - // pick slices from the delta and put them into the new stream - for (; data < dend; data = ndata) - { - ndata = next_delta_info(data, &dc); - - // Data chunks don't need processing - if (dc.data){ - // just copy it over - memcpy((void*)ds, (void*)data, ndata - data); - ds += ndata - data; - num_chunks += 1; - continue; - } - - // Copy Chunks - num_chunks += DIV_copy_slice_to(div, &ds, dc.so, dc.ts); - } - assert(ds - dstream == num_bytes); - assert(num_chunks >= tsi->num_chunks); - assert(DC_rbound(&dc) == tsi->target_size); - - // finally, replace the streams - TSI_replace_stream(tsi, dstream, num_bytes); - tsi->cstart = dstream; // we have NO header ! - assert(tsi->tds == dstream); - tsi->num_chunks = num_chunks; - - - return 1; - -} - -// DELTA CHUNK LIST (PYTHON) -///////////////////////////// -// Internally, it has nothing to do with a ChunkList anymore though -typedef struct { - PyObject_HEAD - // ----------- - ToplevelStreamInfo istream; - -} DeltaChunkList; - - - -int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) -{ - if(args && PySequence_Size(args) > 0){ - PyErr_SetString(PyExc_ValueError, "Too many arguments"); - return -1; - } - - TSI_init(&self->istream); - return 0; -} - - -void DCL_dealloc(DeltaChunkList* self) -{ - TSI_destroy(&(self->istream)); -} - - -PyObject* DCL_py_rbound(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(self->istream.target_size); -} - -// Write using a write function, taking remaining bytes from a base buffer - -PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) -{ - PyObject* pybuf = 0; - PyObject* writeproc = 0; - if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ - PyErr_BadArgument(); - return NULL; - } - - if (!PyObject_CheckReadBuffer(pybuf)){ - PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - - if (!PyCallable_Check(writeproc)){ - PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); - return NULL; - } - - const uchar* base; - Py_ssize_t baselen; - PyObject_AsReadBuffer(pybuf, (const void**)&base, &baselen); - - PyObject* tmpargs = PyTuple_New(1); - - const uchar* data = TSI_first(&self->istream); - const uchar* const dend = TSI_end(&self->istream); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - while (data < dend){ - data = next_delta_info(data, &dc); - DC_apply(&dc, base, writeproc, tmpargs); - } - - Py_DECREF(tmpargs); - Py_RETURN_NONE; -} - -PyMethodDef DCL_methods[] = { - {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, - {NULL} /* Sentinel */ -}; - -PyTypeObject DeltaChunkListType = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "DeltaChunkList", /*tp_name*/ - sizeof(DeltaChunkList), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)DCL_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "Minimal Delta Chunk List",/* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - DCL_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -// Makes a new copy of the DeltaChunkList - you have to do everything yourselve -// in C ... want C++ !! -DeltaChunkList* DCL_new_instance(void) -{ - DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); - assert(dcl); - - DCL_init(dcl, 0, 0); - return dcl; -} - -// Read the next delta chunk from the given stream and advance it -// dc will contain the parsed information, its offset must be set by -// the previous call of next_delta_info, which implies it should remain the -// same instance between the calls. -// Return the altered uchar pointer, reassign it to the input data -inline -const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) -{ - const char cmd = *data++; - - if (cmd & 0x80) - { - uint cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); // this should never get hit with current deltas ... - if (cp_size == 0) cp_size = 0x10000; - - dc->to += dc->ts; - dc->data = NULL; - dc->so = cp_off; - dc->ts = cp_size; - - } else if (cmd) { - // Just share the data - dc->to += dc->ts; - dc->data = data; - dc->ts = cmd; - dc->so = 0; - - data += cmd; - } else { - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - assert(0); - return NULL; - } - - return data; -} - -// Return amount of chunks encoded in the given delta stream -// If read_header is True, then the header msb chunks will be read first. -// Otherwise, the stream is assumed to be scrubbed one past the header -uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) -{ - // read header - if (read_header){ - msb_size(&data, dend); - msb_size(&data, dend); - } - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - uint num_chunks = 0; - - while (data < dend) - { - data = next_delta_info(data, &dc); - num_chunks += 1; - }// END handle command opcodes - - return num_chunks; -} - -PyObject* connect_deltas(PyObject *self, PyObject *dstreams) -{ - // obtain iterator - PyObject* stream_iter = 0; - if (!PyIter_Check(dstreams)){ - stream_iter = PyObject_GetIter(dstreams); - if (!stream_iter){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); - return NULL; - } - } else { - stream_iter = dstreams; - } - - DeltaInfoVector div; - ToplevelStreamInfo tdsinfo; - TSI_init(&tdsinfo); - DIV_init(&div, 0); - - - // GET TOPLEVEL DELTA STREAM - int error = 0; - PyObject* ds = 0; - unsigned int dsi = 0; // delta stream index we process - ds = PyIter_Next(stream_iter); - if (!ds){ - error = 1; - goto _error; - } - - dsi += 1; - tdsinfo.parent_object = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(tdsinfo.parent_object)){ - Py_DECREF(ds); - error = 1; - goto _error; - } - - PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); - if (tdsinfo.tdslen > pow(2, 32)){ - // parent object is deallocated by info structure - Py_DECREF(ds); - PyErr_SetString(PyExc_RuntimeError, "Cannot handle deltas larger than 4GB"); - tdsinfo.parent_object = 0; - - error = 1; - goto _error; - } - Py_DECREF(ds); - - // let it officially know, and initialize its internal state - TSI_set_stream(&tdsinfo, tdsinfo.tds); - - // INTEGRATE ANCESTOR DELTA STREAMS - for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) - { - // Its important to initialize this before the next block which can jump - // to code who needs this to exist ! - PyObject* db = 0; - - // When processing the first delta, we know we will have to alter the tds - // Hence we copy it and deallocate the parent object - if (dsi == 1) { - if (!TSI_copy_stream_from_object(&tdsinfo)){ - PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); - // info structure takes care of the parent_object - error = 1; - goto loop_end; - } - - tdsinfo.num_chunks = compute_chunk_count(tdsinfo.cstart, TSI_end(&tdsinfo), 0); - } - - db = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(db)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); - goto loop_end; - } - - // Fill the stream info structure - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* const dstart = data; - const uchar* const dend = data + dlen; - div.dstream = dstart; - - if (dlen > pow(2, 32)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Cannot currently handle deltas larger than 4GB"); - goto loop_end; - } - - // READ HEADER - msb_size(&data, dend); - const ull target_size = msb_size(&data, dend); - - DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); - - // parse command stream - DeltaInfo* di = 0; // temporary pointer - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - assert(data < dend); - while (data < dend) - { - di = DIV_append(&div); - di->dso = data - dstart; - if ((data = next_delta_info(data, &dc))){ - di->to = dc.to; - } else { - error = 1; - goto loop_end; - } - }// END handle command opcodes - - // finalize information - div.di_last_size = dc.ts; - - if (DC_rbound(&dc) != target_size){ - PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); - error = 1; - } - - #ifdef DEBUG - fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i KiB, target_size = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000, (int)tdsinfo.target_size/1000); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i KiB\n", (int)div.size, (int)div.reserved_size, (int)dlen/1000); - #endif - - if (!DIV_connect_with_base(&tdsinfo, &div)){ - error = 1; - } - - #ifdef DEBUG - fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000); - #endif - - // destroy members, but keep memory - DIV_reset(&div); - -loop_end: - // perform cleanup - Py_DECREF(ds); - Py_DECREF(db); - - if (error){ - break; - } - }// END for each stream object - - if (dsi == 0){ - PyErr_SetString(PyExc_ValueError, "No streams provided"); - } - - -_error: - - if (stream_iter != dstreams){ - Py_DECREF(stream_iter); - } - - - DIV_destroy(&div); - - // Return the actual python object - its just a container - DeltaChunkList* dcl = DCL_new_instance(); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdsinfo would be deallocated by the chunk list - TSI_destroy(&tdsinfo); - error = 1; - } else { - // Plain copy, transfer ownership to dcl - dcl->istream = tdsinfo; - } - - if (error){ - // Will dealloc tdcv - Py_XDECREF(dcl); - return NULL; - } - - return (PyObject*)dcl; -} - - -// Write using a write function, taking remaining bytes from a base buffer -// replaces the corresponding method in python -PyObject* apply_delta(PyObject* self, PyObject* args) -{ - PyObject* pybbuf = 0; - PyObject* pydbuf = 0; - PyObject* pytbuf = 0; - if (!PyArg_ParseTuple(args, "OOO", &pybbuf, &pydbuf, &pytbuf)){ - PyErr_BadArgument(); - return NULL; - } - - PyObject* objects[] = { pybbuf, pydbuf, pytbuf }; - assert(sizeof(objects) / sizeof(PyObject*) == 3); - - uint i; - for(i = 0; i < 3; i++){ - if (!PyObject_CheckReadBuffer(objects[i])){ - PyErr_SetString(PyExc_ValueError, "Argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - } - - Py_ssize_t lbbuf; Py_ssize_t ldbuf; Py_ssize_t ltbuf; - const uchar* bbuf; const uchar* dbuf; - uchar* tbuf; - PyObject_AsReadBuffer(pybbuf, (const void**)(&bbuf), &lbbuf); - PyObject_AsReadBuffer(pydbuf, (const void**)(&dbuf), &ldbuf); - - if (PyObject_AsWriteBuffer(pytbuf, (void**)(&tbuf), <buf)){ - PyErr_SetString(PyExc_ValueError, "Argument 3 must be a writable buffer"); - return NULL; - } - - const uchar* data = dbuf; - const uchar* dend = dbuf + ldbuf; - - while (data < dend) - { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - memcpy(tbuf, bbuf + cp_off, cp_size); - tbuf += cp_size; - - } else if (cmd) { - memcpy(tbuf, data, cmd); - tbuf += cmd; - data += cmd; - } else { - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - return NULL; - } - }// END handle command opcodes - - Py_RETURN_NONE; -} diff --git a/gitdb/_delta_apply.h b/gitdb/_delta_apply.h deleted file mode 100644 index 1fcd53832..000000000 --- a/gitdb/_delta_apply.h +++ /dev/null @@ -1,6 +0,0 @@ -#include - -extern PyObject* connect_deltas(PyObject *self, PyObject *dstreams); -extern PyObject* apply_delta(PyObject* self, PyObject* args); - -extern PyTypeObject DeltaChunkListType; diff --git a/gitdb/_fun.c b/gitdb/_fun.c deleted file mode 100644 index 49970386f..000000000 --- a/gitdb/_fun.c +++ /dev/null @@ -1,107 +0,0 @@ -#include -#include "_delta_apply.h" - -static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) -{ - const unsigned char *sha; - const unsigned int sha_len; - - // Note: self is only set if we are a c type. We emulate an instance method, - // hence we have to get the instance as 'first' argument - - // get instance and sha - PyObject* inst = 0; - if (!PyArg_ParseTuple(args, "Os#", &inst, &sha, &sha_len)) - return NULL; - - if (sha_len != 20) { - PyErr_SetString(PyExc_ValueError, "Sha is not 20 bytes long"); - return NULL; - } - - if( !inst){ - PyErr_SetString(PyExc_ValueError, "Cannot be called without self"); - return NULL; - } - - // read lo and hi bounds - PyObject* fanout_table = PyObject_GetAttrString(inst, "_fanout_table"); - if (!fanout_table){ - PyErr_SetString(PyExc_ValueError, "Couldn't obtain fanout table"); - return NULL; - } - - unsigned int lo = 0, hi = 0; - if (sha[0]){ - PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)(sha[0]-1)); - lo = PyInt_AS_LONG(item); - Py_DECREF(item); - } - PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)sha[0]); - hi = PyInt_AS_LONG(item); - Py_DECREF(item); - item = 0; - - Py_DECREF(fanout_table); - - // get sha query function - PyObject* get_sha = PyObject_GetAttrString(inst, "sha"); - if (!get_sha){ - PyErr_SetString(PyExc_ValueError, "Couldn't obtain sha method"); - return NULL; - } - - PyObject *sha_str = 0; - while (lo < hi) { - const int mid = (lo + hi)/2; - sha_str = PyObject_CallFunction(get_sha, "i", mid); - if (!sha_str) { - return NULL; - } - - // we really trust that string ... for speed - const int cmp = memcmp(PyString_AS_STRING(sha_str), sha, 20); - Py_DECREF(sha_str); - sha_str = 0; - - if (cmp < 0){ - lo = mid + 1; - } - else if (cmp > 0) { - hi = mid; - } - else { - Py_DECREF(get_sha); - return PyInt_FromLong(mid); - }// END handle comparison - }// END while lo < hi - - // nothing found, cleanup - Py_DECREF(get_sha); - Py_RETURN_NONE; -} - -static PyMethodDef py_fun[] = { - { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "See python implementation" }, - { "apply_delta", (PyCFunction)apply_delta, METH_VARARGS, "See python implementation" }, - { NULL, NULL, 0, NULL } -}; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC init_perf(void) -{ - PyObject *m; - - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - - m = Py_InitModule3("_perf", py_fun, NULL); - if (m == NULL) - return; - - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); -} diff --git a/gitdb/fun.py b/gitdb/fun.py index ac9d99395..8ca38c867 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -776,6 +776,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - from _perf import connect_deltas + from gitdb_speedups._perf import connect_deltas except ImportError: pass diff --git a/gitdb/pack.py b/gitdb/pack.py index 2447455b5..20a451549 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -35,7 +35,7 @@ ) try: - from _perf import PackIndexFile_sha_to_index + from gitdb_speedups._perf import PackIndexFile_sha_to_index except ImportError: pass # END try c module diff --git a/gitdb/stream.py b/gitdb/stream.py index be95c113a..2f4c12dfb 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,7 @@ has_perf_mod = False PY26 = sys.version_info[:2] < (2, 7) try: - from _perf import apply_delta as c_apply_delta + from gitdb_speedups._perf import apply_delta as c_apply_delta has_perf_mod = True except ImportError: pass diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..3c6e79cf3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index c2f9f1eb1..eccb95278 100755 --- a/setup.py +++ b/setup.py @@ -1,125 +1,45 @@ -#!/usr/bin/env python -from distutils.core import setup, Extension -from distutils.command.build_py import build_py -from distutils.command.build_ext import build_ext +from setuptools import setup -import os -import sys +# NOTE: This is currently duplicated from the gitdb.__init__ module, because +# that's just how you write a setup.py (nobody reads this stuff out of the +# module) -# wow, this is a mixed bag ... I am pretty upset about all of this ... -setuptools_build_py_module = None -try: - # don't pull it in if we don't have to - if 'setuptools' in sys.modules: - import setuptools.command.build_py as setuptools_build_py_module - from setuptools.command.build_ext import build_ext -except ImportError: - pass - - -class build_ext_nofail(build_ext): - - """Doesn't fail when build our optional extensions""" - - def run(self): - try: - build_ext.run(self) - except Exception: - print("Ignored failure when building extensions, pure python modules will be used instead") - # END ignore errors - - -def get_data_files(self): - """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, - the line dealing with the ``plen`` has a bug which causes it to truncate too much. - It is fixed in the system interpreters as they receive patches, and shows how - bad it is if something doesn't have proper unittests. - The code here is a plain copy of the python2.6 version which works for all. - - Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - - # this one is just for the setup tools ! They don't iniitlialize this variable - # when they should, but do it on demand using this method.Its crazy - if hasattr(self, 'analyze_manifest'): - self.analyze_manifest() - # END handle setuptools ... - - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir) + 1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - -build_py.get_data_files = get_data_files -if setuptools_build_py_module: - setuptools_build_py_module.build_py._get_data_files = get_data_files -# END apply setuptools patch too - -# NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot -# satisfy the dependencies at installation time, unfortunately, due to inherent limitations -# of distutils, which cannot install the prerequesites of a package before the acutal package. __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" version_info = (0, 6, 4) __version__ = '.'.join(str(i) for i in version_info) -setup(cmdclass={'build_ext': build_ext_nofail}, - name="gitdb", - version=__version__, - description="Git Object Database", - author=__author__, - author_email=__contact__, - url=__homepage__, - packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), - package_dir = {'gitdb': 'gitdb'}, - ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], - license = "BSD License", - zip_safe=False, - requires=('smmap (>=0.8.5)', ), - install_requires=('smmap >= 0.8.5'), - long_description = """GitDB is a pure-Python git object database""", - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", - # "Development Status :: 4 - Beta", - "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Operating System :: POSIX", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", -],) +setup( + name="gitdb", + version=__version__, + description="Git Object Database", + author=__author__, + author_email=__contact__, + url=__homepage__, + packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), + license="BSD License", + zip_safe=False, + install_requires=['smmap >= 0.8.5'], + long_description="""GitDB is a pure-Python git object database""", + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + ] +) From 55aa3d79607fdf9123d266612d58a65e04a8622e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Oct 2016 10:31:02 +0200 Subject: [PATCH 0327/1790] doc(README): correct appveyor badge And remove some broken badges. --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ef0b23601..83c674456 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview [![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) -[![Build status](https://ci.appveyor.com/api/projects/status/h8rl7thsr42oc0pf?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) +[![Build status](https://ci.appveyor.com/api/projects/status/kuws846av5lvmugo?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) [![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) @@ -40,8 +40,6 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap -[![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) -[![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) [![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: From 09023b1013e081755dca575f62c35eb93ee8065a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 16 Oct 2016 12:10:19 +0200 Subject: [PATCH 0328/1790] chore(rename): smmap -> smmap2 [skip ci] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6a04827d..2c519e648 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ long_description = "See http://github.com/gitpython-developers/smmap" setup( - name="smmap", + name="smmap2", version=smmap.__version__, description="A pure python implementation of a sliding window memory map manager", author=smmap.__author__, From ac5df7061ee11232346b3d0eb3aa5b43eebc847d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 16 Oct 2016 12:18:20 +0200 Subject: [PATCH 0329/1790] chore(version): set version to 2.0 Just to match the name a bit better. [skip ci] --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index e711bbbf3..8b4059c87 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 9, 0) +version_info = (2, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 38866bc7c4956170c681a62c4508f934ac826469 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 16 Oct 2016 12:23:27 +0200 Subject: [PATCH 0330/1790] chore(rename): gitdb2 v2.0 v2 is chosen to better match the name. --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 6554cf904..bfa083bb3 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 4) +version_info = (2, 0, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 18e4aea23..ac5df7061 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 18e4aea23644ea43657cb2e6846b6aaf78720c27 +Subproject commit ac5df7061ee11232346b3d0eb3aa5b43eebc847d diff --git a/setup.py b/setup.py index eccb95278..f7e3761ab 100755 --- a/setup.py +++ b/setup.py @@ -7,11 +7,11 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 4) +version_info = (2, 0, 0) __version__ = '.'.join(str(i) for i in version_info) setup( - name="gitdb", + name="gitdb2", version=__version__, description="Git Object Database", author=__author__, @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap >= 0.8.5'], + install_requires=['smmap2 >= 2.0.0'], long_description="""GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 6a217abbbf1673ab2e5794a3cc0bc151e16b9bc0 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 22:28:40 +0200 Subject: [PATCH 0331/1790] ci: Test on Appveyor for Windows. --- .appveyor.yml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..2daadaa4d --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,49 @@ +# CI on Windows via appveyor +environment: + + matrix: + ## MINGW + # + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + - PYTHON: "C:\\Miniconda35-x64" + PYTHON_VERSION: "3.5" + IS_CONDA: "yes" + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + + ## Print configuration for debugging. + # + - | + echo %PATH% + uname -a + where python pip pip2 pip3 pip34 + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + + - IF "%IS_CONDA%"=="yes" ( + conda info -a & + conda install --yes --quiet pip + ) + - pip install nose wheel coveralls + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - pip install -e . + +build: false + +test_script: + - IF "%PYTHON_VERSION%"=="3.5" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) From 62202bbbb58379814c44bd26cf662e68d3fa6dbb Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 22:34:51 +0200 Subject: [PATCH 0332/1790] appveyor: Add badge on ankostis repo for testing. --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 1b754d09b..ca51dfaba 100644 --- a/README.rst +++ b/README.rst @@ -43,8 +43,8 @@ Once the clone is complete, please be sure to initialize the submodules using cd gitdb git submodule update --init -Run the tests with - +Run the tests with + nosetests DEVELOPMENT @@ -52,13 +52,12 @@ DEVELOPMENT .. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - +.. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed + :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb - .. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr :target: http://www.issuestats.com/github/gitpython-developers/gitdb - .. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue :target: http://www.issuestats.com/github/gitpython-developers/gitdb From 587dc4ec1311e135d70996a077a2f978e303d3fc Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 23:08:31 +0200 Subject: [PATCH 0333/1790] TCs: fix div-by-zero on elapsed times (appveyor CPU is fast!) --- gitdb/test/performance/test_pack.py | 10 +++++----- gitdb/test/performance/test_pack_streaming.py | 6 +++--- gitdb/test/performance/test_stream.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index bdd2b0a37..fc8d9d54f 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -36,7 +36,7 @@ def test_pack_random_access(self): sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) - print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / (elapsed or 1)), file=sys.stderr) # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info @@ -51,7 +51,7 @@ def test_pack_random_access(self): del(pdb._entities) pdb.entities() print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % - (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) + (ns, len(pdb.entities()), elapsed, ns / (elapsed or 1)), file=sys.stderr) # END for each random mode # query info and streams only @@ -62,7 +62,7 @@ def test_pack_random_access(self): pdb_fun(sha) elapsed = time() - st print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % - (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) + (max_items, pdb_fun.__name__.upper(), elapsed, max_items / (elapsed or 1)), file=sys.stderr) # END for each function # retrieve stream and read all @@ -78,7 +78,7 @@ def test_pack_random_access(self): elapsed = time() - st total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % - (max_items, total_kib, total_kib / elapsed, elapsed, max_items / elapsed), file=sys.stderr) + (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) @skip_on_travis_ci def test_loose_correctness(self): @@ -129,5 +129,5 @@ def test_correctness(self): # END for each entity elapsed = time() - st print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % - (count, crc, elapsed, count / elapsed), file=sys.stderr) + (count, crc, elapsed, count / (elapsed or 1)), file=sys.stderr) # END for each verify mode diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index f805e59fa..76f0f4a07 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -52,14 +52,14 @@ def test_pack_writing(self): # END gather objects for pack-writing elapsed = time() - st print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % - (ni, elapsed, ni / elapsed), file=sys.stderr) + (ni, elapsed, ni / (elapsed or 1)), file=sys.stderr) st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % - (total_kb, elapsed, total_kb / elapsed), sys.stderr) + (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) @skip_on_travis_ci def test_stream_reading(self): @@ -82,4 +82,4 @@ def test_stream_reading(self): elapsed = time() - st total_kib = total_size / 1000 print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % - (ni, total_kib, total_kib / elapsed, elapsed, ni / elapsed), sys.stderr) + (ni, total_kib, total_kib / (elapsed or 1), elapsed, ni / (elapsed or 1)), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index bd66b26ad..704f4d094 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -70,7 +70,7 @@ def test_large_data_streaming(self, path): size_kib = size / 1000 print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % - (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) + (size_kib, fsize_kib, desc, elapsed_add, size_kib / (elapsed_add or 1)), file=sys.stderr) # reading all at once st = time() @@ -81,7 +81,7 @@ def test_large_data_streaming(self, path): stream.seek(0) assert shadata == stream.getvalue() print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % - (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) + (size_kib, desc, elapsed_readall, size_kib / (elapsed_readall or 1)), file=sys.stderr) # reading in chunks of 1 MiB cs = 512 * 1000 @@ -101,7 +101,7 @@ def test_large_data_streaming(self, path): cs_kib = cs / 1000 print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % - (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) + (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) # del db file so we keep something to do os.remove(db_file) From d48679a0b15feae754ebe9ef4c9d5809db0c0d08 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 2 Oct 2016 01:10:47 +0200 Subject: [PATCH 0334/1790] tc: HALF FIX of `test_pack_entity ()` + On Windows, you cannot write onto a file held by another live file-pointer (test_pack.py:#L204). + The TC fails later, on clean up (the usual). --- gitdb/test/test_pack.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 601c0eaba..6e31363bf 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -188,7 +188,8 @@ def test_pack_entity(self, rw_dir): # pack writing - write all packs into one # index path can be None - pack_path = tempfile.mktemp('', "pack", rw_dir) + pack_path1 = tempfile.mktemp('', "pack1", rw_dir) + pack_path2 = tempfile.mktemp('', "pack2", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 @@ -196,7 +197,9 @@ def rewind_streams(): for obj in pack_objs: obj.stream.seek(0) # END utility - for ppath, ipath, num_obj in zip((pack_path, ) * 2, (index_path, None), (len(pack_objs), None)): + for ppath, ipath, num_obj in zip((pack_path1, pack_path2), + (index_path, None), + (len(pack_objs), None)): iwrite = None if ipath: ifile = open(ipath, 'wb') @@ -214,7 +217,7 @@ def rewind_streams(): assert os.path.getsize(ppath) > 100 # verify pack - pf = PackFile(ppath) + pf = PackFile(ppath) # FIXME: Leaks file-pointer(s)! assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha From b2eafcc9c99c8b00a4adc2a316e800a9a7dc7319 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 22 Oct 2016 13:02:49 +0200 Subject: [PATCH 0335/1790] fix(MapWindow): unicode foes in read_into_memory() used by gitpython TCs Drop Windows only codepath bypassing memory-mapping due to some leaks in the past. Now Appveyor proves everything run ok. Additionally, this codepath had unicode problems on PY3. So deleting it, fixes 2 TCs in gitpython: + TestRepo.test_file_handle_leaks() + TestObjDbPerformance.test_random_access() See https://github.com/gitpython-developers/GitPython/issues/525 --- smmap/mman.py | 16 ++++++++++------ smmap/util.py | 25 +++---------------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index af12ee98f..9df69ed29 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -31,9 +31,9 @@ class WindowCursor(object): __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file - '_region', # our current region or None - '_ofs', # relative offset from the actually mapped area to our start area - '_size' # maximum size we should provide + '_region', # our current class:`MapRegion` or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide ) def __init__(self, manager=None, regions=None): @@ -308,10 +308,14 @@ def _collect_lru_region(self, size): if 0, we try to free any available region :return: Amount of freed regions - **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. + .. Note:: + We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. - **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + .. TODO:: + implement a case where all unusued regions are discarded efficiently. + Currently its only brute force + """ num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None diff --git a/smmap/util.py b/smmap/util.py index 36ff1ab89..02df41a13 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -116,16 +116,13 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 + _need_compat_layer = sys.version_info[:2] < (2, 6) if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset # END handle additional slot #{ Configuration - # Used for testing only. If True, all data will be loaded into memory at once. - # This makes sure no file handles will remain open. - _test_read_into_memory = False #} END configuration def __init__(self, path_or_fd, ofs, size, flags=0): @@ -160,10 +157,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0): # bark that the size is too large ... many extra file accesses because # if this ... argh ! actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) - if self._test_read_into_memory: - self._mf = self._read_into_memory(fd, ofs, actual_size) - else: - self._mf = mmap(fd, actual_size, **kwargs) + self._mf = mmap(fd, actual_size, **kwargs) # END handle memory mode self._size = len(self._mf) @@ -179,19 +173,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0): # We assume the first one to use us keeps us around self.increment_client_count() - def _read_into_memory(self, fd, offset, size): - """:return: string data as read from the given file descriptor, offset and size """ - os.lseek(fd, offset, os.SEEK_SET) - mf = '' - bytes_todo = size - while bytes_todo: - chunk = 1024 * 1024 - d = os.read(fd, chunk) - bytes_todo -= len(d) - mf += d - # END loop copy items - return mf - def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) @@ -267,7 +248,7 @@ class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map + '_file_size' # total size of the file we map ) def __new__(cls, path): From 6e55a1c70843e5e5e29686176325f309f8285f29 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 22 Oct 2016 16:28:19 +0200 Subject: [PATCH 0336/1790] chore(version): v2.0.1 Better windows support --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index 8b4059c87..9cfd0a1c1 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (2, 0, 0) +version_info = (2, 0, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From a36fbeb41ce0e29d7f4f9eeb301c60032e47577c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Dec 2016 11:58:31 +0100 Subject: [PATCH 0337/1790] doc(readme): make limitations way more prominent Also inform about the likelyhood of leaking system resources. [skip ci] --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 83c674456..247848d04 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,12 @@ When reading from many possibly large files in a fashion similar to random acces Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. +## Limitations + +* **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor. +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. + ## Overview @@ -33,11 +39,6 @@ For performance critical 64 bit applications, a simplified version of memory map The package was tested on all of the previously mentioned configurations. -## Limitations - -* The memory access is read-only by design. -* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. - ## Installing smmap [![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) From 30329354b370ed6bfac74290ce0c5d2ab17307d1 Mon Sep 17 00:00:00 2001 From: stuertz Date: Sun, 26 Mar 2017 15:45:10 +0200 Subject: [PATCH 0338/1790] Skip Test on Windows Currently renaming files is not supported while the the OS doesn't support renaming open files. When closing the file, as done in the code by using http://smmap.readthedocs.io/en/latest/api.html#smmap.mman.StaticWindowMapManager.force_map_handle_removal_win force_map_handle_removal_win, we can rename, but the cache does still have a handle to this file and crashes. --- gitdb/test/db/test_pack.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index a90158151..e6c2032ca 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -10,13 +10,17 @@ from gitdb.db import PackedDB from gitdb.exc import BadObject, AmbiguousObjectName +from gitdb.util import mman import os import random +import sys +from unittest import skipIf class TestPackDB(TestDBBase): + @skipIf(sys.platform == "win32", "not supported on windows currently") @with_rw_directory @with_packs_rw def test_writing(self, path): @@ -30,6 +34,11 @@ def test_writing(self, path): # packs removed - rename a file, should affect the glob pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" + if sys.platform == "win32": + # This is just the beginning: While using thsi function, we are not + # allowed to have any handle to thsi path, which is currently not + # the case. The pack caching does have a handle :-( + mman.force_map_handle_removal_win(pack_path) os.rename(pack_path, new_pack_path) pdb.update_cache(force=True) From 57c3a4fc59b6babe71859b2bf92b2b2fc909ce2a Mon Sep 17 00:00:00 2001 From: stuertz Date: Sun, 26 Mar 2017 15:48:07 +0200 Subject: [PATCH 0339/1790] Fixed Tests / Code for Windows. Sometimes the OS or some other process has the handle to file a bit longer, and the file could not be deleted immediatly. Retry 10 Times with 100ms distance. --- gitdb/test/performance/test_stream.py | 4 ++-- gitdb/util.py | 27 +++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 704f4d094..bd8953e15 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -9,7 +9,7 @@ from gitdb.db import LooseObjectDB from gitdb import IStream -from gitdb.util import bin_to_hex +from gitdb.util import bin_to_hex, remove from gitdb.fun import chunk_size from time import time @@ -104,5 +104,5 @@ def test_large_data_streaming(self, path): (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) # del db file so we keep something to do - os.remove(db_file) + remove(db_file) # END for each randomization factor diff --git a/gitdb/util.py b/gitdb/util.py index 242be4405..95ab9b28c 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -6,6 +6,7 @@ import os import mmap import sys +import time import errno from io import BytesIO @@ -58,7 +59,6 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir isfile = os.path.isfile rename = os.rename -remove = os.remove dirname = os.path.dirname basename = os.path.basename join = os.path.join @@ -67,6 +67,25 @@ def unpack_from(fmt, data, offset=0): close = os.close fsync = os.fsync + +def _retry(func, *args, **kwargs): + # Wrapper around functions, that are problematic on "Windows". Sometimes + # the OS or someone else has still a handle to the file + if sys.platform == "win32": + for _ in xrange(10): + try: + return func(*args, **kwargs) + except Exception: + time.sleep(0.1) + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + + +def remove(*args, **kwargs): + return _retry(os.remove, *args, **kwargs) + + # Backwards compatibility imports from gitdb.const import ( NULL_BIN_SHA, @@ -321,7 +340,7 @@ def open(self, write=False, stream=False): self._fd = os.open(self._filepath, os.O_RDONLY | binary) except: # assure we release our lockfile - os.remove(self._lockfilepath()) + remove(self._lockfilepath()) raise # END handle lockfile # END open descriptor for reading @@ -365,7 +384,7 @@ def _end_writing(self, successful=True): # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": if isfile(self._filepath): - os.remove(self._filepath) + remove(self._filepath) # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) @@ -376,7 +395,7 @@ def _end_writing(self, successful=True): chmod(self._filepath, int("644", 8)) else: # just delete the file so far, we failed - os.remove(lockfile) + remove(lockfile) # END successful handling #} END utilities From 060e5ea18b010e4d8441a5cfad699aba69593c72 Mon Sep 17 00:00:00 2001 From: stuertz Date: Sun, 26 Mar 2017 22:50:56 +0200 Subject: [PATCH 0340/1790] Release the file handle, before deleting, otherwise win fails. --- gitdb/test/performance/test_stream.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index bd8953e15..92d28e4a3 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -104,5 +104,6 @@ def test_large_data_streaming(self, path): (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) # del db file so we keep something to do + ostream = None # To release the file handle (win) remove(db_file) # END for each randomization factor From d6c097eaf759dabfd3c42b55958962b6e9a05063 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:20:48 +0200 Subject: [PATCH 0341/1790] close smmap handles, to be able to delete files / trees in process (req. for windows) --- gitdb/pack.py | 12 ++++++++++++ gitdb/test/test_pack.py | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 20a451549..115d94365 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -266,6 +266,10 @@ def __init__(self, indexpath): super(PackIndexFile, self).__init__() self._indexpath = indexpath + def close(self): + mman.force_map_handle_removal_win(self._indexpath) + self._cursor = None + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -527,6 +531,10 @@ class PackFile(LazyMixin): def __init__(self, packpath): self._packpath = packpath + def close(self): + mman.force_map_handle_removal_win(self._packpath) + self._cursor = None + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() @@ -668,6 +676,10 @@ def __init__(self, pack_or_index_path): self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + def close(self): + self._index.close() + self._pack.close() + def _set_cache_(self, attr): # currently this can only be _offset_map # TODO: make this a simple sorted offset array which can be bisected diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 6e31363bf..24e2a3134 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -217,10 +217,11 @@ def rewind_streams(): assert os.path.getsize(ppath) > 100 # verify pack - pf = PackFile(ppath) # FIXME: Leaks file-pointer(s)! + pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha + pf.close() # verify index if ipath is not None: @@ -231,6 +232,7 @@ def rewind_streams(): assert idx.packfile_checksum() == pack_sha assert idx.indexfile_checksum() == index_sha assert idx.size() == len(pack_objs) + idx.close() # END verify files exist # END for each packpath, indexpath pair @@ -245,7 +247,8 @@ def rewind_streams(): # END for each crc mode # END for each info assert count == len(pack_objs) - + entity.close() + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack From 891ac5126540dcb087242f9bb0cadd02ca021324 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:36:43 +0200 Subject: [PATCH 0342/1790] Use range instead of xrange, good enough here --- gitdb/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/util.py b/gitdb/util.py index 95ab9b28c..8a1819b6d 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -72,7 +72,7 @@ def _retry(func, *args, **kwargs): # Wrapper around functions, that are problematic on "Windows". Sometimes # the OS or someone else has still a handle to the file if sys.platform == "win32": - for _ in xrange(10): + for _ in range(10): try: return func(*args, **kwargs) except Exception: From bcdffc46e3993d7743b90982009eec49df667ab4 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:44:14 +0200 Subject: [PATCH 0343/1790] fixed to be py26 compat --- gitdb/test/db/test_pack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index e6c2032ca..f6e275126 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -16,14 +16,16 @@ import random import sys -from unittest import skipIf +from nose.plugins.skip import SkipTest class TestPackDB(TestDBBase): - @skipIf(sys.platform == "win32", "not supported on windows currently") @with_rw_directory @with_packs_rw def test_writing(self, path): + if sys.platform == "win32": + raise SkipTest("FIXME: Currently fail on windows") + pdb = PackedDB(path) # on demand, we init our pack cache From 7bced788880015075754ce3645cef3a351166ff4 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:46:23 +0200 Subject: [PATCH 0344/1790] Typos --- gitdb/test/db/test_pack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f6e275126..9694238bd 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -37,9 +37,9 @@ def test_writing(self, path): pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" if sys.platform == "win32": - # This is just the beginning: While using thsi function, we are not - # allowed to have any handle to thsi path, which is currently not - # the case. The pack caching does have a handle :-( + # While using this function, we are not allowed to have any handle + # to this path, which is currently not the case. The pack caching + # does still have a handle :-( mman.force_map_handle_removal_win(pack_path) os.rename(pack_path, new_pack_path) From ce7889ef31c41cf5ecdb468a69f7e195b0ea5826 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 28 May 2017 17:59:03 +0200 Subject: [PATCH 0345/1790] chore(version-up): v2.0.2 Include python 3.5/3.6 in list of supported versions, provide signed archives. --- Makefile | 18 +++++++++++++++--- setup.py | 2 ++ smmap/__init__.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ca3051dd4..061588245 100644 --- a/Makefile +++ b/Makefile @@ -12,13 +12,14 @@ all: $(info sdist) doc: - cd docs && make html + cd doc && make html clean-docs: - cd docs && make clean + cd doc && make clean clean-files: git clean -fx + rm -rf build/ dist/ clean: clean-files clean-docs @@ -34,4 +35,15 @@ build: sdist: ./setup.py sdist - +release: clean + # Check if latest tag is the current head we're releasing + echo "Latest tag = $$(git tag | sort -nr | head -n1)" + echo "HEAD SHA = $$(git rev-parse head)" + echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + make force_release + +force_release: clean + git push --tags + python setup.py sdist bdist_wheel + twine upload -s -i byronimo@gmail.com dist/* diff --git a/setup.py b/setup.py index 2c519e648..800fe4fd1 100755 --- a/setup.py +++ b/setup.py @@ -50,6 +50,8 @@ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", ], long_description=long_description, tests_require=('nose', 'nosexcover'), diff --git a/smmap/__init__.py b/smmap/__init__.py index 9cfd0a1c1..dd9b0781a 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (2, 0, 1) +version_info = (2, 0, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 813166d9a93eff6ea2140c2ba1714aa83feb1d4c Mon Sep 17 00:00:00 2001 From: Joseph LaFreniere Date: Wed, 7 Jun 2017 20:53:57 -0500 Subject: [PATCH 0346/1790] Add MANIFEST.in to include LICENSE in distribution From https://packaging.python.org/distributing/#manifest-in: > A "MANIFEST.in" is needed in certain cases where you need to package > additional files that python setup.py sdist (or bdist_wheel) don't > automatically include. --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..399a207e6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +# Include the license file +include LICENSE From 91d506e120d4a0f98cbef202325e301c632445c5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 10 Jun 2017 18:49:59 +0200 Subject: [PATCH 0347/1790] chore(version-up): v2.0.3 --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index dd9b0781a..b252c211c 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (2, 0, 2) +version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 3add9cea092514a82931108bbc05a15355900a52 Mon Sep 17 00:00:00 2001 From: wangweichen Date: Thu, 13 Jul 2017 13:55:49 +0800 Subject: [PATCH 0348/1790] fix open encoding error. --- gitdb/db/ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 2e3db86db..84f9f6cab 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -41,7 +41,7 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - with open(self._ref_file, 'r') as f: + with open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] except (OSError, IOError): pass From 5e0fea5f6b9e47f52d045d0449dd1a09943496a0 Mon Sep 17 00:00:00 2001 From: wangweichen Date: Thu, 13 Jul 2017 14:34:09 +0800 Subject: [PATCH 0349/1790] change codecs to open with support encoding="utf-8" --- gitdb/db/ref.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 84f9f6cab..94a2f01f8 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,6 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php +import codecs from gitdb.db.base import ( CompoundDB, ) @@ -41,7 +42,7 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - with open(self._ref_file, 'r', encoding="utf-8") as f: + with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] except (OSError, IOError): pass From 0d5062eac9d03ea63975446439600e63feb83163 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 28 Sep 2017 10:52:51 +0200 Subject: [PATCH 0350/1790] Upgrade makefile --- Makefile | 16 +++++++++++++++- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c6c159bdc..8cb323e9f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,21 @@ SETUP = $(PYTHON) setup.py TESTRUNNER = $(shell which nosetests) TESTFLAGS = -all: build +all:: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +release:: clean + # Check if latest tag is the current head we're releasing + echo "Latest tag = $$(git tag | sort -nr | head -n1)" + echo "HEAD SHA = $$(git rev-parse head)" + echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + make force_release + +force_release:: clean + git push --tags + python3 setup.py sdist bdist_wheel + twine upload -s -i byronimo@gmail.com dist/* doc:: make -C doc/ html diff --git a/gitdb/__init__.py b/gitdb/__init__.py index bfa083bb3..e184e4b1f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 0) +version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index ac5df7061..91d506e12 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit ac5df7061ee11232346b3d0eb3aa5b43eebc847d +Subproject commit 91d506e120d4a0f98cbef202325e301c632445c5 diff --git a/setup.py b/setup.py index f7e3761ab..4bff11dff 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 0) +version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) setup( From ed3ecbbe7b05433f6c70410a3f1fda1ae85508a3 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Aug 2017 09:46:50 -0500 Subject: [PATCH 0351/1790] Update homepage --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index b252c211c..725c420ea 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -2,7 +2,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" -__homepage__ = "https://github.com/Byron/smmap" +__homepage__ = "https://github.com/gitpython-developers/smmap" version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) From 90c4f25493b918ff9dc4ee52ae8216a554bb3446 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 29 Sep 2017 13:16:49 +0200 Subject: [PATCH 0352/1790] Add python 3.6 to the testing suite --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ba0beaa7b..7288a2acb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) git: From a4ca69e9c93d00185dc729e68f2ef86ffe138410 Mon Sep 17 00:00:00 2001 From: Michael Overmeyer Date: Mon, 5 Mar 2018 20:06:45 +0000 Subject: [PATCH 0353/1790] Switched broken pypip.in badges to shields.io --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ca51dfaba..917b403b0 100644 --- a/README.rst +++ b/README.rst @@ -6,10 +6,10 @@ GitDB allows you to access bare git repositories for reading and writing. It aim Installation ============ -.. image:: https://pypip.in/version/gitdb/badge.svg +.. image:: https://img.shields.io/pypi/v/gitdb.svg :target: https://pypi.python.org/pypi/gitdb/ :alt: Latest Version -.. image:: https://pypip.in/py_versions/gitdb/badge.svg +.. image:: https://img.shields.io/pypi/pyversions/gitdb.svg :target: https://pypi.python.org/pypi/gitdb/ :alt: Supported Python versions .. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest From 857196a7312209eed5c84fca857ab6c486bcbd6b Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 21:29:54 +0300 Subject: [PATCH 0354/1790] Drop support for EOL Python --- .travis.yml | 2 -- gitdb/util.py | 5 +---- setup.py | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7288a2acb..1341a1d9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: python python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" diff --git a/gitdb/util.py b/gitdb/util.py index 8a1819b6d..d680f9766 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -19,10 +19,7 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -if sys.version_info < (2, 6): - mman = StaticWindowMapManager() -else: - mman = SlidingWindowMapManager() +mman = SlidingWindowMapManager() # END handle mman import hashlib diff --git a/setup.py b/setup.py index 4bff11dff..931596472 100755 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ zip_safe=False, install_requires=['smmap2 >= 2.0.0'], long_description="""GitDB is a pure-Python git object database""", + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -34,11 +35,8 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", ] From 140104a7ecb3dfb5c43b57c4829b9d7142b4aeaf Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 21:32:56 +0300 Subject: [PATCH 0355/1790] Upgrade Python syntax with pyupgrade https://github.com/asottile/pyupgrade --- gitdb/db/pack.py | 2 +- gitdb/db/ref.py | 2 +- gitdb/test/lib.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 6b03d8383..1e37d738e 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -148,7 +148,7 @@ def update_cache(self, force=False): # packs are supposed to be prefixed with pack- by git-convention # get all pack files, figure out what changed pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) - our_pack_files = set(item[1].pack().path() for item in self._entities) + our_pack_files = {item[1].pack().path() for item in self._entities} # new packs for pack_file in (pack_files - our_pack_files): diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 94a2f01f8..2bb1de790 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -49,7 +49,7 @@ def _update_dbs_from_ref_file(self): # END handle alternates ref_paths_set = set(ref_paths) - cur_ref_paths_set = set(db.root_path() for db in self._dbs) + cur_ref_paths_set = {db.root_path() for db in self._dbs} # remove existing for path in (cur_ref_paths_set - ref_paths_set): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index bbdd241cd..ab1842dbd 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -86,7 +86,7 @@ def wrapper(self): try: return func(self, path) except Exception: - sys.stderr.write("Test %s.%s failed, output is at %r\n" % (type(self).__name__, func.__name__, path)) + sys.stderr.write("Test {}.{} failed, output is at {!r}\n".format(type(self).__name__, func.__name__, path)) keep = True raise finally: From 186db667f703b695c2914040e98d7977f6e61272 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 21:36:22 +0300 Subject: [PATCH 0356/1790] Python 3.6 is tested and supported --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 931596472..27bb75429 100755 --- a/setup.py +++ b/setup.py @@ -39,5 +39,6 @@ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", ] ) From 3e28e6e896543814ea312baa9ba6c6065caba797 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 13 Oct 2018 12:41:52 +0200 Subject: [PATCH 0357/1790] Bump patch level: remove support for old python versions --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index e184e4b1f..344c3de01 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 27bb75429..ea6ba189f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) setup( From b1adf606f416f82ec69cd83cfc2b94cddc6928bd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 13 Oct 2018 12:45:29 +0200 Subject: [PATCH 0358/1790] Another version bump... dunno what happened there. --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 344c3de01..a2d26245b 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index ea6ba189f..27eb65c8e 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 5) __version__ = '.'.join(str(i) for i in version_info) setup( From 54a34e8976587762df7b02ef09b0150c065fb9e1 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 22:04:27 +0300 Subject: [PATCH 0359/1790] Drop support for EOL Python --- .travis.yml | 5 ----- README.md | 3 +-- doc/source/intro.rst | 3 +-- setup.py | 6 ++---- smmap/buf.py | 43 ++++++++++++++----------------------------- smmap/util.py | 38 +------------------------------------- tox.ini | 2 +- 7 files changed, 20 insertions(+), 80 deletions(-) diff --git a/.travis.yml b/.travis.yml index 464fafb2e..ff77ede15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,6 @@ language: python python: - # These versions are unsupported by travis, even though smmap claims to still support these outdated versions - # - 2.4 - # - 2.5 - - 2.6 - 2.7 - - 3.3 - 3.4 - 3.5 install: diff --git a/README.md b/README.md index 247848d04..08702930e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Although memory maps have many advantages, they represent a very limited system * **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor. * The memory access is read-only by design. -* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. ## Overview @@ -34,7 +33,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.4, 2.5, 2.6, 2.7 or 3.3 +* Python 2.7 or 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 15f5bf08a..8f7cf3c26 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.4, 2.5, 2.6, 2.7 or 3.3 +* Python 2.7 or 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. @@ -31,7 +31,6 @@ The package was tested on all of the previously mentioned configurations. Limitations ########### * The memory access is read-only by design. -* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. ################ Installing smmap diff --git a/setup.py b/setup.py index 800fe4fd1..23347a3e3 100755 --- a/setup.py +++ b/setup.py @@ -13,12 +13,12 @@ if os.path.exists("README.md"): long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/gitpython-developers/smmap" + long_description = "See https://github.com/gitpython-developers/smmap" setup( name="smmap2", version=smmap.__version__, - description="A pure python implementation of a sliding window memory map manager", + description="A pure Python implementation of a sliding window memory map manager", author=smmap.__author__, author_email=smmap.__contact__, url=smmap.__homepage__, @@ -45,10 +45,8 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", diff --git a/smmap/buf.py b/smmap/buf.py index 438292b60..786775a8b 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -88,35 +88,20 @@ def __getslice__(self, i, j): # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code pyvers = sys.version_info[:2] - if (3, 0) <= pyvers <= (3, 3): - # Memory view cannot be joined below python 3.4 ... - out = bytes() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - # This is slower than the join ... but what can we do ... - out += d - del(d) - # END while there are bytes to read - return out - else: - md = list() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - # Make sure we don't keep references, as c.use_region() might attempt to free resources, but - # can't unless we use pure bytes - if hasattr(d, 'tobytes'): - d = d.tobytes() - md.append(d) - # END while there are bytes to read - return bytes().join(md) + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() + md.append(d) + # END while there are bytes to read + return bytes().join(md) # END fast or slow path #{ Interface diff --git a/smmap/util.py b/smmap/util.py index 02df41a13..defef1f32 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,13 +3,7 @@ import sys from mmap import mmap, ACCESS_READ -try: - from mmap import ALLOCATIONGRANULARITY -except ImportError: - # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly - # useful for aligning the offset. The offset argument doesn't exist there though - from mmap import PAGESIZE as ALLOCATIONGRANULARITY -# END handle pythons missing quality assurance +from mmap import ALLOCATIONGRANULARITY __all__ = ["align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] @@ -116,11 +110,6 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[:2] < (2, 6) - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - # END handle additional slot #{ Configuration #} END configuration @@ -147,11 +136,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0): kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because @@ -161,10 +145,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0): # END handle memory mode self._size = len(self._mf) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, self._size) - # END handle buffer wrapping finally: if isinstance(path_or_fd, string_types()): os.close(fd) @@ -224,22 +204,6 @@ def release(self): """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" self._mf.close() - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return self._size - self._b - - def ofs_end(self): - # always the size - we are as large as it gets - return self._size - - def buffer(self): - return self._mfb - - def includes_ofs(self, ofs): - return self._b <= ofs < self._size - # END handle compat layer - #} END interface diff --git a/tox.ini b/tox.ini index 35b813753..74ca3481d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py26, py27, py33, py34 +envlist = flake8, py27, py34 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From 68a94764aeffe9b90f82f82bbf7d166cfdfacc87 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 22:07:12 +0300 Subject: [PATCH 0360/1790] Add python_requires to help pip --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 23347a3e3..9136339ae 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", classifiers=[ # Picked from # http://pypi.python.org/pypi?:action=list_classifiers From 48e9e30b0ef3c24ac7ed88e6e3bfa37dc945bf4c Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 22:07:54 +0300 Subject: [PATCH 0361/1790] Test Python 3.6 --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff77ede15..9cccb75cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - 2.7 - 3.4 - 3.5 + - 3.6 install: - pip install coveralls script: diff --git a/tox.ini b/tox.ini index 74ca3481d..d1f558bc0 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py27, py34 +envlist = flake8, py27, py34, py35, py36 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From 53944ab34f18d05135ab4b507e718eccfe699248 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 13 Oct 2018 12:56:00 +0200 Subject: [PATCH 0362/1790] bump patch; remove old python versions --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index 725c420ea..1728b832f 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (2, 0, 3) +version_info = (2, 0, 5) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 2f075234b84511ee63b74e69ccc8ddc331de1199 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 15 Oct 2018 10:51:19 +0100 Subject: [PATCH 0363/1790] Tell PyPI long_description is Markdown Rendering of the README on PyPI is broken because it expects RST by default and Markdown is being uploaded. This fixes it. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9136339ae..09755dac5 100755 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ "Programming Language :: Python :: 3.6", ], long_description=long_description, + long_description_content_type='text/markdown', tests_require=('nose', 'nosexcover'), test_suite='nose.collector' ) From 0e4b57d4511686d1aeb5479a7aa0dad3a5338d6e Mon Sep 17 00:00:00 2001 From: xarx00 Date: Fri, 5 Apr 2019 10:46:53 +0200 Subject: [PATCH 0364/1790] Fix for UnicodeEncodeError in git.Repo.clone_from() when path contains non-ascii characters --- gitdb/utils/encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 4d270af9d..d8fd59a4a 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -8,7 +8,7 @@ text_type = unicode -def force_bytes(data, encoding="ascii"): +def force_bytes(data, encoding="utf-8"): if isinstance(data, bytes): return data From a0060cfdc9166bb0b3104e8015faf0689aa6daf1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Jul 2019 18:45:04 +0800 Subject: [PATCH 0365/1790] Associate smmap2 on pypi with this repository Fixes #43 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 08702930e..c7a38173c 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ Issues can be filed on github: * https://github.com/gitpython-developers/smmap/issues +A link to the pypi page related to this repository: + +* https://pypi.org/project/smmap2/ + ## License Information From 79b705f061b51dc151a00729b722fbdebde59f5c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 25 Sep 2019 21:00:30 +0300 Subject: [PATCH 0366/1790] loose: rename only if needed Our user was experiencing issue [1] when using a git repository on NTFS mount running on Linux. The current check checks if we are running on Windows, but it should really check if we are on NTFS. And since checking fs type is not that trivial and not efficient, it is simpler and better to just always apply NTFS-specific logic, since it works on other filesystems as well. [1] https://github.com/iterative/dvc/issues/1880#issuecomment-483253764 --- gitdb/db/loose.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 192c524af..53ade9f79 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -225,16 +225,12 @@ def store(self, istream): if not isdir(obj_dir): mkdir(obj_dir) # END handle destination directory - # rename onto existing doesn't work on windows - if os.name == 'nt': - if isfile(obj_path): - remove(tmp_path) - else: - rename(tmp_path, obj_path) - # end rename only if needed + # rename onto existing doesn't work on NTFS + if isfile(obj_path): + remove(tmp_path) else: rename(tmp_path, obj_path) - # END handle win32 + # end rename only if needed # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr From d77bd023a61419effe77184c52ccf3e19afa6f60 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 28 Sep 2019 13:21:35 +0200 Subject: [PATCH 0367/1790] Bump version --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index e184e4b1f..344c3de01 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 91d506e12..a0060cfdc 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 91d506e120d4a0f98cbef202325e301c632445c5 +Subproject commit a0060cfdc9166bb0b3104e8015faf0689aa6daf1 diff --git a/setup.py b/setup.py index 27bb75429..ea6ba189f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) setup( From 43e16318e9ab95f146e230afe0a7cbdc848473fe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 28 Sep 2019 13:28:50 +0200 Subject: [PATCH 0368/1790] bump version again... --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 344c3de01..4e884076b 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 6) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index ea6ba189f..6a2174fce 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 6) __version__ = '.'.join(str(i) for i in version_info) setup( From 7729239951b5561f5bb5c2d5152ff76833b11826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 8 Jan 2020 09:36:55 +0100 Subject: [PATCH 0369/1790] Fix deprecated calls for Python 3.9 The array methods fromstring/tostring have been deprecated since Python 3.2. Python 3.9 removes them completely. This was discovered when trying to build gitdb package for Fedora 33. https://bugzilla.redhat.com/show_bug.cgi?id=1788660 --- gitdb/pack.py | 2 +- gitdb/test/lib.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 115d94365..2ad2324eb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -410,7 +410,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.frombytes(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more if sys.byteorder == 'little': diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ab1842dbd..42b9ddc7e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -157,7 +157,7 @@ def make_bytes(size_in_bytes, randomize=False): random.shuffle(producer) # END randomize a = array('i', producer) - return a.tostring() + return a.tobytes() def make_object(type, data): From c880f6b0550770eee559091d6276a2e2b097a83a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 09:41:05 +0800 Subject: [PATCH 0370/1790] don't test python 2.7 anymore, support is dropped --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1341a1d9d..17d63804d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.7" - "3.4" - "3.5" - "3.6" From 2f9a799a6c9d125012bb09473bbfe6110f2a7391 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 09:59:38 +0800 Subject: [PATCH 0371/1790] remove appveyor It is slow, it fails, and windows support seems unmaintained, besides always having been an incredible time sink. Thanks to everyone who brought GitDb to where it is right now, and I am happy to bring windows testing back if a maintainer can be found. --- .appveyor.yml | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 2daadaa4d..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -# CI on Windows via appveyor -environment: - - matrix: - ## MINGW - # - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - - PYTHON: "C:\\Miniconda35-x64" - PYTHON_VERSION: "3.5" - IS_CONDA: "yes" - -install: - - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - ## Print configuration for debugging. - # - - | - echo %PATH% - uname -a - where python pip pip2 pip3 pip34 - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - - - IF "%IS_CONDA%"=="yes" ( - conda info -a & - conda install --yes --quiet pip - ) - - pip install nose wheel coveralls - - ## For commits performed with the default user. - - | - git config --global user.email "travis@ci.com" - git config --global user.name "Travis Runner" - - - pip install -e . - -build: false - -test_script: - - IF "%PYTHON_VERSION%"=="3.5" ( - nosetests -v --with-coverage - ) ELSE ( - nosetests -v - ) From df73d7f6874ff11be1b09f65c8dc425671bb924e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 10:01:11 +0800 Subject: [PATCH 0372/1790] Release 3.0.0 --- gitdb/__init__.py | 2 +- setup.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 4e884076b..c9c827970 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 6) +version_info = (3, 0, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 6a2174fce..a66267e48 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 6) +version_info = (3, 0, 0) __version__ = '.'.join(str(i) for i in version_info) setup( @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap2 >= 2.0.0'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -34,11 +34,10 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7" ] ) From e6ee8bf864c726a5461600de28d64c1f06f4e163 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 10:03:50 +0800 Subject: [PATCH 0373/1790] Change package signature to the only yubikey I can use right now --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8cb323e9f..82c0a3bc8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i byronimo@gmail.com dist/* + twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* doc:: make -C doc/ html From d6d1550a1e8dc327d5b310228a66f25a59d6ce9f Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 15 Feb 2020 14:19:38 -0600 Subject: [PATCH 0374/1790] Remove badges for no longer existing Issue Stats site from README --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 917b403b0..9febff0ed 100644 --- a/README.rst +++ b/README.rst @@ -56,10 +56,6 @@ DEVELOPMENT :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb -.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr - :target: http://www.issuestats.com/github/gitpython-developers/gitdb -.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue - :target: http://www.issuestats.com/github/gitpython-developers/gitdb The library is considered mature, and not under active development. It's primary (known) use is in git-python. From 58bce6bd1051e4fd5df7c5c8123a1783cc6e9f84 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:06:35 -0600 Subject: [PATCH 0375/1790] Remove and replace compat.izip --- gitdb/fun.py | 4 ++-- gitdb/pack.py | 3 +-- gitdb/utils/compat.py | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 8ca38c867..3a2248fa1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,7 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import izip, buffer, xrange, PY3 +from gitdb.utils.compat import buffer, xrange, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -314,7 +314,7 @@ def check_integrity(self, target_size=-1): right.next() # this is very pythonic - we might have just use index based access here, # but this could actually be faster - for lft, rgt in izip(left, right): + for lft, rgt in zip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair diff --git a/gitdb/pack.py b/gitdb/pack.py index 2ad2324eb..748df3880 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -63,7 +63,6 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import ( - izip, buffer, xrange, to_bytes @@ -696,7 +695,7 @@ def _set_cache_(self, attr): iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) next(iter_offsets_plus_one) - consecutive = izip(iter_offsets, iter_offsets_plus_one) + consecutive = zip(iter_offsets, iter_offsets_plus_one) offset_map = dict(consecutive) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index a7899cb14..586f3bb9d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -3,11 +3,9 @@ PY3 = sys.version_info[0] == 3 try: - from itertools import izip xrange = xrange except ImportError: # py3 - izip = zip xrange = range # end handle python version From 73a9f7965139e319446c04bbcc9794a8db0de45a Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:07:51 -0600 Subject: [PATCH 0376/1790] Remove and replace izip in TestPack --- gitdb/test/test_pack.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 24e2a3134..dd1c8302b 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -27,11 +27,6 @@ from gitdb.util import to_bin_sha from gitdb.utils.compat import xrange -try: - from itertools import izip -except ImportError: - izip = zip - from nose import SkipTest import os @@ -155,7 +150,7 @@ def test_pack_entity(self, rw_dir): pack_objs.extend(entity.stream_iter()) count = 0 - for info, stream in izip(entity.info_iter(), entity.stream_iter()): + for info, stream in zip(entity.info_iter(), entity.stream_iter()): count += 1 assert info.binsha == stream.binsha assert len(info.binsha) == 20 From 4a692fdd43e67810509b1c1843fa203714b14f0e Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:13:00 -0600 Subject: [PATCH 0377/1790] Remove and replace compat.xrange --- gitdb/db/pack.py | 3 +-- gitdb/fun.py | 4 ++-- gitdb/pack.py | 9 ++++----- gitdb/test/db/lib.py | 3 +-- gitdb/test/lib.py | 3 +-- gitdb/test/performance/test_pack.py | 3 +-- gitdb/test/test_pack.py | 3 +-- gitdb/utils/compat.py | 7 ------- 8 files changed, 11 insertions(+), 24 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 1e37d738e..177ed7bb2 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -18,7 +18,6 @@ ) from gitdb.pack import PackEntity -from gitdb.utils.compat import xrange from functools import reduce @@ -107,7 +106,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in xrange(index.size()): + for index in range(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/fun.py b/gitdb/fun.py index 3a2248fa1..7203de978 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,7 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import buffer, xrange, PY3 +from gitdb.utils.compat import buffer, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -264,7 +264,7 @@ def compress(self): # if first_data_index is not None: nd = StringIO() # new data so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i - 1): + for x in range(first_data_index, i - 1): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data diff --git a/gitdb/pack.py b/gitdb/pack.py index 748df3880..f01055493 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -64,7 +64,6 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import ( buffer, - xrange, to_bytes ) @@ -206,7 +205,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[byte_ord(t[0][0])] += 1 # END prepare fanout - for i in xrange(255): + for i in range(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i + 1] += v @@ -375,7 +374,7 @@ def _read_fanout(self, byte_offset): d = self._cursor.map() out = list() append = out.append - for i in xrange(256): + for i in range(256): append(unpack_from('>L', d, byte_offset + i * 4)[0]) # END for each entry return out @@ -416,7 +415,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in xrange(self.size())) + return tuple(self.offset(index) for index in range(self.size())) # END handle version def sha_to_index(self, sha): @@ -715,7 +714,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in xrange(self._index.size()): + for index in range(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 528bcc144..c6f4316cd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,6 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type -from gitdb.utils.compat import xrange from io import BytesIO @@ -45,7 +44,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in xrange(ni): + for i in range(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 42b9ddc7e..a04084f5f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,7 +4,6 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import OStream -from gitdb.utils.compat import xrange import sys import random @@ -151,7 +150,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes // 4 - producer = xrange(actual_size) + producer = range(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index fc8d9d54f..b59d5a971 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -17,7 +17,6 @@ from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB -from gitdb.utils.compat import xrange from gitdb.test.lib import skip_on_travis_ci import sys @@ -118,7 +117,7 @@ def test_correctness(self): for entity in pdb.entities(): pack_verify = entity.is_valid_stream sha_by_index = entity.index().sha - for index in xrange(entity.index().size()): + for index in range(entity.index().size()): try: assert pack_verify(sha_by_index(index), use_crc=crc) count += 1 diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index dd1c8302b..8bf78f0d2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -25,7 +25,6 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from gitdb.utils.compat import xrange from nose import SkipTest @@ -58,7 +57,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in xrange(index.size()): + for oidx in range(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 586f3bb9d..99e7ae654 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -2,13 +2,6 @@ PY3 = sys.version_info[0] == 3 -try: - xrange = xrange -except ImportError: - # py3 - xrange = range -# end handle python version - try: # Python 2 buffer = buffer From 3d14cc84b0a5e42d755d24d77a6e4af6bea8c3c1 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:36:03 -0600 Subject: [PATCH 0378/1790] Remove and replace compat.buffer --- gitdb/fun.py | 8 ++++---- gitdb/pack.py | 11 ++++------- gitdb/stream.py | 5 ++--- gitdb/utils/compat.py | 10 ---------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 7203de978..92b8b1c70 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,7 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import buffer, PY3 +from gitdb.utils.compat import PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -101,7 +101,7 @@ def delta_chunk_apply(dc, bbuf, write): :param write: write method to call with data to write""" if dc.data is None: # COPY DATA FROM SOURCE - write(buffer(bbuf, dc.so, dc.ts)) + write(bbuf[dc.so:dc.so + dc.ts]) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -698,7 +698,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (rbound < cp_size or rbound > src_buf_size): break - write(buffer(src_buf, cp_off, cp_size)) + write(src_buf[cp_off:cp_off + cp_size]) elif c: write(db[i:i + c]) i += c @@ -741,7 +741,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (rbound < cp_size or rbound > src_buf_size): break - write(buffer(src_buf, cp_off, cp_size)) + write(src_buf[cp_off:cp_off + cp_size]) elif c: write(db[i:i + c]) i += c diff --git a/gitdb/pack.py b/gitdb/pack.py index f01055493..68da2b7fb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -62,10 +62,7 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import ( - buffer, - to_bytes -) +from gitdb.utils.compat import to_bytes import tempfile import array @@ -117,7 +114,7 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + stream = DecompressMemMapReader(data[total_rela_offset:], False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: @@ -408,7 +405,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.frombytes(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.frombytes(self._cursor.map()[self._pack_offset:self._pack_64_offset]) # networkbyteorder to something array likes more if sys.byteorder == 'little': @@ -836,7 +833,7 @@ def is_valid_stream(self, sha, use_crc=False): while cur_pos < next_offset: rbound = min(cur_pos + chunk_size, next_offset) size = rbound - cur_pos - this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + this_crc_value = crc_update(pack_data[cur_pos:cur_pos + size], this_crc_value) cur_pos += size # END window size loop diff --git a/gitdb/stream.py b/gitdb/stream.py index 2f4c12dfb..b94ef245d 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -27,7 +27,6 @@ ) from gitdb.const import NULL_BYTE, BYTE_SPACE -from gitdb.utils.compat import buffer from gitdb.utils.encoding import force_bytes has_perf_mod = False @@ -278,7 +277,7 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) + indata = self._m[self._cws:self._cwe] # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) @@ -414,7 +413,7 @@ def _set_cache_brute_(self, attr): buf = dstream.read(512) # read the header information + X offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) + buffer_info_list.append((buf[offset:], offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 99e7ae654..8c7c06bd7 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,22 +4,12 @@ try: # Python 2 - buffer = buffer memoryview = buffer # Assume no memory view ... def to_bytes(i): return i except NameError: # Python 3 has no `buffer`; only `memoryview` - # However, it's faster to just slice the object directly, maybe it keeps a view internally - def buffer(obj, offset, size=None): - if size is None: - # return memoryview(obj)[offset:] - return obj[offset:] - else: - # return memoryview(obj)[offset:offset+size] - return obj[offset:offset + size] - # end buffer reimplementation # smmap can return memory view objects, which can't be compared as buffers/bytes can ... def to_bytes(i): if isinstance(i, memoryview): From d1c20d559a1f264dad12d1033a052ffc1c159260 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:37:34 -0600 Subject: [PATCH 0379/1790] Remove compat.memoryview --- gitdb/utils/compat.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 8c7c06bd7..d5791edcf 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,20 +4,15 @@ try: # Python 2 - memoryview = buffer - # Assume no memory view ... def to_bytes(i): return i except NameError: - # Python 3 has no `buffer`; only `memoryview` # smmap can return memory view objects, which can't be compared as buffers/bytes can ... def to_bytes(i): if isinstance(i, memoryview): return i.tobytes() return i - memoryview = memoryview - try: MAXSIZE = sys.maxint except AttributeError: From 59c053ee05d6e5f50f8699260aa0e362b567c033 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:54:24 -0600 Subject: [PATCH 0380/1790] Remove and replace compat.to_bytes --- gitdb/pack.py | 7 +++++-- gitdb/utils/compat.py | 11 ----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 68da2b7fb..a38468e37 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -62,7 +62,6 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import to_bytes import tempfile import array @@ -877,7 +876,11 @@ def collect_streams_at_offset(self, offset): stream = streams[-1] while stream.type_id in delta_types: if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(to_bytes(stream.delta_info)) + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + if isinstance(stream.delta_info, memoryview): + sindex = self._index.sha_to_index(stream.delta_info.tobytes()) + else: + sindex = self._index.sha_to_index(stream.delta_info) if sindex is None: break stream = self._pack.stream(self._index.offset(sindex)) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index d5791edcf..6909c53e0 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -2,17 +2,6 @@ PY3 = sys.version_info[0] == 3 -try: - # Python 2 - def to_bytes(i): - return i -except NameError: - # smmap can return memory view objects, which can't be compared as buffers/bytes can ... - def to_bytes(i): - if isinstance(i, memoryview): - return i.tobytes() - return i - try: MAXSIZE = sys.maxint except AttributeError: From 321c3b46b4792cf83bf0c5814d5a1a43cdf6933d Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:57:23 -0600 Subject: [PATCH 0381/1790] Remove and replace compat.MAXSIZE --- gitdb/db/loose.py | 4 ++-- gitdb/utils/compat.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 53ade9f79..7bf92dacf 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -50,11 +50,11 @@ stream_copy ) -from gitdb.utils.compat import MAXSIZE from gitdb.utils.encoding import force_bytes import tempfile import os +import sys __all__ = ('LooseObjectDB', ) @@ -196,7 +196,7 @@ def store(self, istream): if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version - stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size) + stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 6909c53e0..c4264748d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -1,8 +1,3 @@ import sys PY3 = sys.version_info[0] == 3 - -try: - MAXSIZE = sys.maxint -except AttributeError: - MAXSIZE = sys.maxsize From c41268071bba593cecd420c400603e094f40a6dc Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 08:01:00 -0600 Subject: [PATCH 0382/1790] Remove and replace encoding.string_types --- gitdb/utils/encoding.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 4d270af9d..156f01677 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -1,10 +1,8 @@ from gitdb.utils import compat if compat.PY3: - string_types = (str, ) text_type = str else: - string_types = (basestring, ) text_type = unicode @@ -12,7 +10,7 @@ def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data - if isinstance(data, string_types): + if isinstance(data, str): return data.encode(encoding) return data From 77dc809542d15c40dbe60ff55cd830082c3ad904 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 08:02:08 -0600 Subject: [PATCH 0383/1790] Remove and replace encoding.text_type --- gitdb/utils/encoding.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 156f01677..25ebeadd7 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -1,11 +1,3 @@ -from gitdb.utils import compat - -if compat.PY3: - text_type = str -else: - text_type = unicode - - def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -17,13 +9,10 @@ def force_bytes(data, encoding="ascii"): def force_text(data, encoding="utf-8"): - if isinstance(data, text_type): + if isinstance(data, str): return data if isinstance(data, bytes): return data.decode(encoding) - if compat.PY3: - return text_type(data, encoding) - else: - return text_type(data) + return str(data, encoding) From db9a65e3b0eceb9c52359273afb3313860e5e322 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 08:11:54 -0600 Subject: [PATCH 0384/1790] Remove compat.PY3 --- gitdb/fun.py | 211 ++++++++++++++---------------------------- gitdb/utils/compat.py | 3 - 2 files changed, 67 insertions(+), 147 deletions(-) delete mode 100644 gitdb/utils/compat.py diff --git a/gitdb/fun.py b/gitdb/fun.py index 92b8b1c70..98465975d 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,6 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -424,20 +423,12 @@ def pack_object_header_info(data): type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size - if PY3: - while c & 0x80: - c = byte_ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop - else: - while c & 0x80: - c = ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop # end performance at expense of maintenance ... return (type_id, size, i) @@ -450,28 +441,16 @@ def create_pack_object_header(obj_type, obj_size): :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte - if PY3: - hdr = bytearray() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr.append(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - # END until size is consumed - hdr.append(c) - else: - hdr = bytes() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - # END until size is consumed - hdr += chr(c) + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + # END until size is consumed + hdr.append(c) # end handle interpreter return hdr @@ -484,26 +463,15 @@ def msb_size(data, offset=0): i = 0 l = len(data) hit_msb = False - if PY3: - while i < l: - c = data[i + offset] - size |= (c & 0x7f) << i * 7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range - else: - while i < l: - c = ord(data[i + offset]) - size |= (c & 0x7f) << i * 7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range + while i < l: + c = data[i + offset] + size |= (c & 0x7f) << i * 7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") @@ -663,93 +631,48 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf - if PY3: - while i < delta_buf_size: - c = db[i] - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = db[i] - i += 1 - if (c & 0x02): - cp_off |= (db[i] << 8) - i += 1 - if (c & 0x04): - cp_off |= (db[i] << 16) - i += 1 - if (c & 0x08): - cp_off |= (db[i] << 24) - i += 1 - if (c & 0x10): - cp_size = db[i] - i += 1 - if (c & 0x20): - cp_size |= (db[i] << 8) - i += 1 - if (c & 0x40): - cp_size |= (db[i] << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(src_buf[cp_off:cp_off + cp_size]) - elif c: - write(db[i:i + c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - else: - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(src_buf[cp_off:cp_off + cp_size]) - elif c: - write(db[i:i + c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - # end save byte_ord call and prevent performance regression in py2 + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(src_buf[cp_off:cp_off + cp_size]) + elif c: + write(db[i:i + c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py deleted file mode 100644 index c4264748d..000000000 --- a/gitdb/utils/compat.py +++ /dev/null @@ -1,3 +0,0 @@ -import sys - -PY3 = sys.version_info[0] == 3 From 5dd0f302f101e66d9d70a3b17ce0f379b4db214b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 17 Feb 2020 09:15:48 +0800 Subject: [PATCH 0385/1790] bump version to 3.0.1 --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 22deb6db3..9d3b2dca2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +3.0.1 +***** + +* removed all python2 compatibility shims, GitDB now is a Python 3 program. + ***** 0.6.1 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c9c827970..6fa31e470 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 0) +version_info = (3, 0, 1) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index a66267e48..12cebcd66 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 0) +version_info = (3, 0, 1) __version__ = '.'.join(str(i) for i in version_info) setup( From f356e12766480852d0e30ae7b786cdf5f24d8cea Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 17 Feb 2020 11:16:43 +0800 Subject: [PATCH 0386/1790] Now with PR: 3.0.2 --- doc/source/changes.rst | 2 +- gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 9d3b2dca2..aa7a8901c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ***** -3.0.1 +3.0.2 ***** * removed all python2 compatibility shims, GitDB now is a Python 3 program. diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 6fa31e470..5a52cf205 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 1) +version_info = (3, 0, 2) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 12cebcd66..a66b229f3 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 1) +version_info = (3, 0, 2) __version__ = '.'.join(str(i) for i in version_info) setup( From 02de02cbd938015ff6ba3e668da4e641fdd74c4a Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 22 Feb 2020 16:25:56 -0600 Subject: [PATCH 0387/1790] Restrict smmap2 version to <3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a66b229f3..49cbf3650 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap2 >= 2.0.0'], + install_requires=['smmap2>=2,<3'], long_description="""GitDB is a pure-Python git object database""", python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From 139811a89279482c4df9cddb7d7e69d2e2c36c47 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 22 Feb 2020 16:26:11 -0600 Subject: [PATCH 0388/1790] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed4898ec2..92b887293 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=0.8.3 +smmap2>=2,<3 From 09aa35b62ed341124a7b4757acf35b849a7a39ad Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 22 Feb 2020 18:15:58 -0600 Subject: [PATCH 0389/1790] Improve changelog for v3.0.2 --- doc/source/changes.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index aa7a8901c..e4d4ebc14 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -6,7 +6,8 @@ Changelog 3.0.2 ***** -* removed all python2 compatibility shims, GitDB now is a Python 3 program. +* Removed Python 2 compatibility shims + (`#56 `_) ***** 0.6.1 From 541472db1dcae538875f95216f1655652aa30181 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 23 Feb 2020 09:31:01 +0800 Subject: [PATCH 0390/1790] Change package name to 'smmap'; bump version to 3.0.0 https://github.com/gitpython-developers/smmap/issues/44 --- README.md | 2 +- setup.py | 2 +- smmap/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c7a38173c..175bc3445 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Issues can be filed on github: A link to the pypi page related to this repository: -* https://pypi.org/project/smmap2/ +* https://pypi.org/project/smmap/ ## License Information diff --git a/setup.py b/setup.py index 9136339ae..ea02f457b 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ long_description = "See https://github.com/gitpython-developers/smmap" setup( - name="smmap2", + name="smmap", version=smmap.__version__, description="A pure Python implementation of a sliding window memory map manager", author=smmap.__author__, diff --git a/smmap/__init__.py b/smmap/__init__.py index 1728b832f..2bdf05543 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (2, 0, 5) +version_info = (3, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 1b92f4f4a4ff2bd1057b2b3fcf472e78c3d7abc3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 23 Feb 2020 09:33:52 +0800 Subject: [PATCH 0391/1790] Update makefile to sign with new signature --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 061588245..328c836b9 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ release: clean @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" make force_release -force_release: clean +force_release:: clean git push --tags - python setup.py sdist bdist_wheel - twine upload -s -i byronimo@gmail.com dist/* + python3 setup.py sdist bdist_wheel + twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* From 74aef73b0dd1f97b89f95f67500ae9c4c405ff15 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 00:00:54 -0600 Subject: [PATCH 0392/1790] v3.0.3.post1 --- doc/source/changes.rst | 15 +++++++++++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e4d4ebc14..ac4b477d4 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,21 @@ Changelog ######### +*********** +3.0.3.post1 +*********** + +* Fixed changelogs for v3.0.2 and v3.0.3 + +***** +3.0.3 +***** + +* Changed ``force_bytes`` to use UTF-8 encoding by default + (`#49 `_) +* Restricted smmap2 version requirement to < 3 +* Updated requirements.txt + ***** 3.0.2 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5a52cf205..b6a0341c8 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 2) +version_info = (3, 0, 3, "post1") __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 49cbf3650..b79dcb29f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 2) +version_info = (3, 0, 3, "post1") __version__ = '.'.join(str(i) for i in version_info) setup( From 72698cc333654430a2cc8bd103dc5994d95b3f1b Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 08:27:32 -0600 Subject: [PATCH 0393/1790] Remove carriage returns in setup.py long_description Mitigates https://github.com/pypa/setuptools/issues/1440 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b71e6795..f0c67ad2d 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ import smmap if os.path.exists("README.md"): - long_description = codecs.open('README.md', "r", "utf-8").read() + long_description = codecs.open('README.md', "r", "utf-8").read().replace('\r\n', '\n') else: long_description = "See https://github.com/gitpython-developers/smmap" From d076f665dae16bd03eeb9df862860d54968eda84 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 08:32:24 -0600 Subject: [PATCH 0394/1790] v3.0.1 --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f99e85fb7..e9c2662ef 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +****** +v3.0.1 +****** +- Switched back to the smmap package name on PyPI and fixed the smmap2 mirror package + (`#44 `_) +- Fixed setup.py ``long_description`` rendering + (`#40 `_) + ********** v0.9.0 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 2bdf05543..40861f5be 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 0) +version_info = (3, 0, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 253dfe7092f83229d9e99059e7c51f678a557fd2 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 09:13:36 -0600 Subject: [PATCH 0395/1790] v4.0.1 --- doc/source/changes.rst | 10 ++++++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- requirements.txt | 2 +- setup.py | 6 +++--- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ac4b477d4..236b9543e 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,16 @@ Changelog ######### +***** +4.0.1 +***** + +* Switched back to the gitdb package name on PyPI and fixed the gitdb2 mirror package + (`#59 `_) +* Switched back to require smmap package and fixed version requirement to >= 3.0.1, < 4 + (`#59 `_) +* Updated smmap submodule + *********** 3.0.3.post1 *********** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index b6a0341c8..5d68ccbe9 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 3, "post1") +version_info = (4, 0, 1) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index a0060cfdc..d076f665d 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit a0060cfdc9166bb0b3104e8015faf0689aa6daf1 +Subproject commit d076f665dae16bd03eeb9df862860d54968eda84 diff --git a/requirements.txt b/requirements.txt index 92b887293..b6ccf50fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap2>=2,<3 +smmap>=3.0.1,<4 diff --git a/setup.py b/setup.py index b79dcb29f..7617583eb 100755 --- a/setup.py +++ b/setup.py @@ -7,11 +7,11 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 3, "post1") +version_info = (4, 0, 1) __version__ = '.'.join(str(i) for i in version_info) setup( - name="gitdb2", + name="gitdb", version=__version__, description="Git Object Database", author=__author__, @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap2>=2,<3'], + install_requires=['smmap>=3.0.1,<4'], long_description="""GitDB is a pure-Python git object database""", python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From 02d83a8bad286eba61de54616f1df183825800de Mon Sep 17 00:00:00 2001 From: Nicusor97 Date: Mon, 24 Feb 2020 17:56:18 +0200 Subject: [PATCH 0396/1790] Remove setup.cfg because the gitdb dropped Python 2 support --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3c6e79cf3..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 From 9b0309553be36cd188b3fd7d2ab95e7497a5e80d Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 24 Feb 2020 10:45:10 -0600 Subject: [PATCH 0397/1790] v4.0.2 --- doc/source/changes.rst | 7 +++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 236b9543e..1e91121da 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +4.0.2 +***** + +* Updated to release as Pure Python Wheel rather than Universal Wheel + (`#62 `_) + ***** 4.0.1 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5d68ccbe9..df1f242d4 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 1) +version_info = (4, 0, 2) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 7617583eb..d1100991a 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 1) +version_info = (4, 0, 2) __version__ = '.'.join(str(i) for i in version_info) setup( From 77b39e915ef32ca83fb61276128e41e5d7886dc2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 13:57:34 +0800 Subject: [PATCH 0398/1790] Create pythonpackage.yml --- .github/workflows/pythonpackage.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/pythonpackage.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 000000000..4c67fbf5b --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1000 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with nose + run: | + pip install nose + ulimit -n 48 + ulimit -n + nosetests -v From cc21afae16d00291ea97cc1f159a4c268b94a198 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:08:42 +0800 Subject: [PATCH 0399/1790] Replace travis with github actions --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9febff0ed..05511a716 100644 --- a/README.rst +++ b/README.rst @@ -50,8 +50,8 @@ Run the tests with DEVELOPMENT =========== -.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master - :target: https://travis-ci.org/gitpython-developers/gitdb +.. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg + :target: https://github.com/gitpython-developers/gitdb/actions .. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png From e6cc702bf8fdd5e742ed77d607e7b697d27bcfef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:09:05 +0800 Subject: [PATCH 0400/1790] don't try to use __file__ when using pyoxidizer --- gitdb/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index df1f242d4..2b06b24bb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -13,7 +13,8 @@ def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('smmap',): - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + if 'PYOXIDIZER' not in os.environ: + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) try: __import__(module) From aa1275255741ce55722a72c932c45fc43e501d58 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:16:01 +0800 Subject: [PATCH 0401/1790] Add github actions --- .github/workflows/pythonpackage.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/pythonpackage.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 000000000..4c67fbf5b --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1000 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with nose + run: | + pip install nose + ulimit -n 48 + ulimit -n + nosetests -v From b22f159593594111c513daddf3ad55c5bb0ba7f9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:19:26 +0800 Subject: [PATCH 0402/1790] Fix github actions - no dependencies need to be installed --- .github/workflows/pythonpackage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4c67fbf5b..4f06a18d9 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -28,7 +28,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - name: Lint with flake8 run: | pip install flake8 From 2ceedc91958dc080b31e426150610eef52774f5f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:21:10 +0800 Subject: [PATCH 0403/1790] Remove basestring reference, no py2 support --- smmap/util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index defef1f32..13604c581 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,11 +23,7 @@ def buffer(obj, offset, size): def string_types(): - if sys.version_info[0] >= 3: - return str - else: - return basestring - + return str def align_to_mmap(num, round_up): """ From ebec89621ed48b5cba97b695fda28389fafaa0a1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:23:23 +0800 Subject: [PATCH 0404/1790] Update badges to represent reality --- .appveyor.yml | 1 + .travis.yml | 1 + README.md | 4 +--- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 2daadaa4d..2090a0569 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,3 +1,4 @@ +# NOT USED, just for reference. See github actions for CI configuration # CI on Windows via appveyor environment: diff --git a/.travis.yml b/.travis.yml index 9cccb75cf..50e5b97e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +# NOT USED, just for reference. See github actions for CI configuration language: python python: - 2.7 diff --git a/README.md b/README.md index 175bc3445..780168e31 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview -[![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) -[![Build status](https://ci.appveyor.com/api/projects/status/kuws846av5lvmugo?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) -[![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) +![Python package](https://github.com/gitpython-developers/smmap/workflows/Python%20package/badge.svg) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) From f4d7a58b4d96200cd057a38a0758d3c84901f57e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:25:21 +0800 Subject: [PATCH 0405/1790] bump patch level --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e9c2662ef..e5b0e79ce 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v3.0.2 +****** + +- signed release + ****** v3.0.1 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 40861f5be..72df0cf3a 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 1) +version_info = (3, 0, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 57d3f7544820f9fd4202f4ebd0f198c43c8e575d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:35:01 +0800 Subject: [PATCH 0406/1790] bump patch level; mark travis-ci as unused --- .travis.yml | 1 + gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17d63804d..b980d36e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +# NOT USED, just for reference. See github actions for CI configuration language: python python: - "3.4" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index d076f665d..f4d7a58b4 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit d076f665dae16bd03eeb9df862860d54968eda84 +Subproject commit f4d7a58b4d96200cd057a38a0758d3c84901f57e diff --git a/setup.py b/setup.py index d1100991a..e8f8d68c4 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 2) +version_info = (4, 0, 3) __version__ = '.'.join(str(i) for i in version_info) setup( From 0002408bc03033ed90041d04c4547343a1505537 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:37:37 +0800 Subject: [PATCH 0407/1790] update changelog; remove sublime text (editor) configuration --- doc/source/changes.rst | 6 ++++ etc/sublime-text/gitdb.sublime-project | 44 -------------------------- 2 files changed, 6 insertions(+), 44 deletions(-) delete mode 100644 etc/sublime-text/gitdb.sublime-project diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 1e91121da..3c116c309 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +4.0.3 +***** + +* Support for PyOxidizer + ***** 4.0.2 ***** diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project deleted file mode 100644 index d0e2e5132..000000000 --- a/etc/sublime-text/gitdb.sublime-project +++ /dev/null @@ -1,44 +0,0 @@ -{ - "folders": - [ - // GITDB - //////// - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "gitdb/ext", - "dist", - "doc/build", - ".tox" - ] - }, - // SMMAP - //////// - { - "follow_symlinks": true, - "path": "../../gitdb/ext/smmap", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "dist", - "doc/build", - ".tox" - ] - }, - ] -} From b8ab128b7ff8dc951ac42c1fd57d61c26864680a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:40:07 +0800 Subject: [PATCH 0408/1790] =?UTF-8?q?Bump=20patch=20again;=20=E2=80=A6uplo?= =?UTF-8?q?ad=20error=20due=20to=20state=20on=20disk,=20yikes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/source/changes.rst | 2 +- gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 3c116c309..69a6dba98 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ***** -4.0.3 +4.0.4 ***** * Support for PyOxidizer diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2b06b24bb..5f56773d0 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 2) +version_info = (4, 0, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index e8f8d68c4..48d641a8c 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 3) +version_info = (4, 0, 4) __version__ = '.'.join(str(i) for i in version_info) setup( From 4f08b0a2bb6715b1f143782c64e18dc33bb77487 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:44:35 +0800 Subject: [PATCH 0409/1790] Remove windows build badge [skip CI] --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 05511a716..13dad89fc 100644 --- a/README.rst +++ b/README.rst @@ -52,8 +52,6 @@ DEVELOPMENT .. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg :target: https://github.com/gitpython-developers/gitdb/actions -.. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed - :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb From 919695d4e4101237a8d2c4fb65807a42b7ff63d4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:45:44 +0800 Subject: [PATCH 0410/1790] Also remove coveralls [skip CI] --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 13dad89fc..44b3eddf8 100644 --- a/README.rst +++ b/README.rst @@ -52,8 +52,6 @@ DEVELOPMENT .. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg :target: https://github.com/gitpython-developers/gitdb/actions -.. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png - :target: https://coveralls.io/r/gitpython-developers/gitdb The library is considered mature, and not under active development. It's primary (known) use is in git-python. From 5a42c622639b12977f8735043f059704c869427d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:46:28 +0800 Subject: [PATCH 0411/1790] Remove issuestats, which doesn't exist anymore [skip CI] --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 780168e31..fcb9217f2 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,6 @@ Although memory maps have many advantages, they represent a very limited system ## Overview ![Python package](https://github.com/gitpython-developers/smmap/workflows/Python%20package/badge.svg) -[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) -[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. From 24668c4e2463aad775edb9d23732d5d11d0e7754 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 19:16:11 +0800 Subject: [PATCH 0412/1790] Change required key to 2CF*, which seems to be the only good one --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 328c836b9..b867ad70c 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,4 @@ release: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* + twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* From 09f96f289dbb674e64668bcb0a088aae9dff2a29 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 19:17:40 +0800 Subject: [PATCH 0413/1790] re-release with different signing key --- doc/source/changes.rst | 2 +- smmap/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e5b0e79ce..f859b6076 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ****** -v3.0.2 +v3.0.3 ****** - signed release diff --git a/smmap/__init__.py b/smmap/__init__.py index 72df0cf3a..e7e43d8cd 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 2) +version_info = (3, 0, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From a5d3d7e7ec4e2b52c93509bdf35999d66f91e06d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 19:41:43 +0800 Subject: [PATCH 0414/1790] change package signing key back to what it was --- Makefile | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 82c0a3bc8..a5098b754 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* + twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* doc:: make -C doc/ html diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f4d7a58b4..09f96f289 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f4d7a58b4d96200cd057a38a0758d3c84901f57e +Subproject commit 09f96f289dbb674e64668bcb0a088aae9dff2a29 From 5e5f940dff80beaa3eedf9342ef502f5e630d5ed Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 May 2020 11:37:02 +0800 Subject: [PATCH 0415/1790] Bump patch level (to create a new correctly signed release) --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f859b6076..f8ddb1da0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v3.0.4 +****** + +- signed release (with correct key this time) + ****** v3.0.3 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index e7e43d8cd..a6671e104 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 3) +version_info = (3, 0, 4) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 163f2649e5a5f7b8ba03fc1714bf4693b1a015d0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 May 2020 11:45:15 +0800 Subject: [PATCH 0416/1790] Bump patch level for creating a new properly signed release --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 69a6dba98..05748a08d 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +4.0.5 +***** + +* Re-release of 4.0.4, with known signature + ***** 4.0.4 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5f56773d0..813fe5af6 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 4) +version_info = (4, 0, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 09f96f289..5e5f940df 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 09f96f289dbb674e64668bcb0a088aae9dff2a29 +Subproject commit 5e5f940dff80beaa3eedf9342ef502f5e630d5ed diff --git a/setup.py b/setup.py index 48d641a8c..facdf3d73 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 4) +version_info = (4, 0, 5) __version__ = '.'.join(str(i) for i in version_info) setup( From e5410b4166d177f90901db4986753787d34bc48f Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Fri, 12 Jun 2020 13:52:38 +0300 Subject: [PATCH 0417/1790] Fix exception causes in loose.py --- gitdb/db/loose.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 7bf92dacf..a63a2ef33 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -138,12 +138,12 @@ def _map_loose_object(self, sha): # try again without noatime try: return file_contents_ro_filepath(db_path) - except OSError: - raise BadObject(sha) + except OSError as new_e: + raise BadObject(sha) from new_e # didn't work because of our flag, don't try it again self._fd_open_flags = 0 else: - raise BadObject(sha) + raise BadObject(sha) from e # END handle error # END exception handling From 112252cef0d418fd070671e64b18558c2f2cf2f1 Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Sun, 14 Jun 2020 14:35:48 +0300 Subject: [PATCH 0418/1790] Fix exception causes all over the codebase --- gitdb/__init__.py | 4 ++-- gitdb/db/mem.py | 4 ++-- gitdb/util.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5f56773d0..31e4d4583 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -18,8 +18,8 @@ def _init_externals(): try: __import__(module) - except ImportError: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + except ImportError as e: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) from e # END verify import # END handel imports diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 871133468..5b242e46e 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -74,8 +74,8 @@ def stream(self, sha): # rewind stream for the next one to read ostream.stream.seek(0) return ostream - except KeyError: - raise BadObject(sha) + except KeyError as e: + raise BadObject(sha) from e # END exception handling def size(self): diff --git a/gitdb/util.py b/gitdb/util.py index d680f9766..c4cafecc6 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -326,8 +326,8 @@ def open(self, write=False, stream=False): else: self._fd = fd # END handle file descriptor - except OSError: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + except OSError as e: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) from e # END handle lock retrieval # open actual file if required From c96c755fa30277fbaadf79603a0b4fa1054ce2cb Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Mon, 15 Jun 2020 10:48:01 +0300 Subject: [PATCH 0419/1790] Add Ram Rachum to AUTHORS --- AUTHORS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS b/AUTHORS index 490baad8e..6c7e9b997 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,4 @@ Creator: Sebastian Thiel + +Contributors: + - Ram Rachum (@cool-RR) From 56cbfa002a44dce471b58ca3e6680d89ba9ec8a0 Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:31:13 -0600 Subject: [PATCH 0420/1790] Fix changelog version number v3.0.3 was skipped --- doc/source/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f8ddb1da0..3169674de 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -9,7 +9,7 @@ v3.0.4 - signed release (with correct key this time) ****** -v3.0.3 +v3.0.2 ****** - signed release From 0b53ddc88a1221a9b933bc53570729c96cb4f09d Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:44:30 -0600 Subject: [PATCH 0421/1790] Remove Sublime Text project definition --- etc/sublime-text/smmap.sublime-project | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 etc/sublime-text/smmap.sublime-project diff --git a/etc/sublime-text/smmap.sublime-project b/etc/sublime-text/smmap.sublime-project deleted file mode 100644 index 251ebbd28..000000000 --- a/etc/sublime-text/smmap.sublime-project +++ /dev/null @@ -1,21 +0,0 @@ -{ - "folders": - [ - // SMMAP - //////// - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - ] - }, - ] -} From e79fe25f16c3a8bfcf315b269412f0927486155d Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:48:06 -0600 Subject: [PATCH 0422/1790] Remove duplicate sys import --- smmap/buf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 786775a8b..cdbfad3a8 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -3,8 +3,6 @@ __all__ = ["SlidingWindowMapBuffer"] -import sys - try: bytes except NameError: From 066f038cfc74d6095289b5dc3f8810a242e81ecb Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:53:34 -0600 Subject: [PATCH 0423/1790] Remove bytes existence check Python < 2.6 is no longer supported --- smmap/buf.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index cdbfad3a8..18766daf8 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -3,11 +3,6 @@ __all__ = ["SlidingWindowMapBuffer"] -try: - bytes -except NameError: - bytes = str - class SlidingWindowMapBuffer(object): From 93c7ba67f0614416482916741f24829bef06e22f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:21:10 +0800 Subject: [PATCH 0424/1790] Revert "Remove basestring reference, no py2 support" This reverts commit 2ceedc91958dc080b31e426150610eef52774f5f. --- smmap/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index 13604c581..defef1f32 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,7 +23,11 @@ def buffer(obj, offset, size): def string_types(): - return str + if sys.version_info[0] >= 3: + return str + else: + return basestring + def align_to_mmap(num, round_up): """ From 06c4972c87dcd07bb42071b86c2b0d2aae03e581 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:06:50 -0600 Subject: [PATCH 0425/1790] Ignore Flake8 undefined name error for Python 2 support --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index defef1f32..3d7329158 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -26,7 +26,7 @@ def string_types(): if sys.version_info[0] >= 3: return str else: - return basestring + return basestring # noqa: F821 def align_to_mmap(num, round_up): From c661324746a26df609d1ace2106a1c8b916cf802 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:12:51 -0600 Subject: [PATCH 0426/1790] Improve changelog for v3.0.2 --- doc/source/changes.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 3169674de..f92db64ad 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -12,7 +12,8 @@ v3.0.4 v3.0.2 ****** -- signed release +- Signed release +- Switched to GitHub Actions for CI ****** v3.0.1 From 37cc3c09c188b65996e170cddce2d151bf682388 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:14:03 -0600 Subject: [PATCH 0427/1790] Improve capitalization consistency in changelog for v3.0.4 --- doc/source/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f92db64ad..bc1db5d59 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -6,7 +6,7 @@ Changelog v3.0.4 ****** -- signed release (with correct key this time) +- Signed release (with correct key this time) ****** v3.0.2 From 65f171aadffb2d83124065fc6c193f133936643b Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:17:49 -0600 Subject: [PATCH 0428/1790] Add changelog for v3.0.5 --- doc/source/changes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index bc1db5d59..c9665b673 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v3.0.5 +****** + +- Restored Python 2 support removed in v3.0.2 + ****** v3.0.4 ****** From 119cc417541f400b3533c02d53d3d5f236e87023 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Jan 2021 09:59:05 +0800 Subject: [PATCH 0429/1790] bump patch level --- Makefile | 2 +- doc/source/changes.rst | 2 ++ smmap/__init__.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b867ad70c..593a758ee 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,4 @@ release: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* + twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* diff --git a/doc/source/changes.rst b/doc/source/changes.rst index c9665b673..041711bae 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -7,6 +7,8 @@ v3.0.5 ****** - Restored Python 2 support removed in v3.0.2 +- changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. See + https://keybase.io/byronbates for proof of ownership. ****** v3.0.4 diff --git a/smmap/__init__.py b/smmap/__init__.py index a6671e104..abc09d952 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 4) +version_info = (3, 0, 5) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 9c2570bf07db6dfc7d616b5ffaee84cc33b5c0f5 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:01:32 -0600 Subject: [PATCH 0430/1790] Improve changelog for v3.0.5 --- doc/source/changes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 041711bae..bc0b1b63c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -7,8 +7,8 @@ v3.0.5 ****** - Restored Python 2 support removed in v3.0.2 -- changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. See - https://keybase.io/byronbates for proof of ownership. +- Changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. + See https://keybase.io/byronbates for proof of ownership. ****** v3.0.4 From 341b27890a0e1353ccdf08b345aee9f5934d1847 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:08:03 -0600 Subject: [PATCH 0431/1790] Remove string_types --- smmap/mman.py | 3 +-- smmap/util.py | 11 ++--------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9df69ed29..d7dfe6d62 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MapRegion, MapRegionList, is_64_bit, - string_types, buffer, ) @@ -227,7 +226,7 @@ def fd(self): **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), string_types()): + if isinstance(self._rlist.path_or_fd(), str): raise ValueError("File descriptor queried although mapping was generated from path") # END handle type return self._rlist.path_or_fd() diff --git a/smmap/util.py b/smmap/util.py index 3d7329158..4e713f8ae 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -22,13 +22,6 @@ def buffer(obj, offset, size): # return obj[offset:offset + size] -def string_types(): - if sys.version_info[0] >= 3: - return str - else: - return basestring # noqa: F821 - - def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. @@ -146,7 +139,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0): self._size = len(self._mf) finally: - if isinstance(path_or_fd, string_types()): + if isinstance(path_or_fd, str): os.close(fd) # END only close it if we opened it # END close file handle @@ -229,7 +222,7 @@ def path_or_fd(self): def file_size(self): """:return: size of file we manager""" if self._file_size is None: - if isinstance(self._path_or_fd, string_types()): + if isinstance(self._path_or_fd, str): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size From da0a610bd9a050239299d37ce40d4c01d5cacc81 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:15:53 -0600 Subject: [PATCH 0432/1790] Remove buffer --- smmap/mman.py | 3 +-- smmap/util.py | 13 +------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index d7dfe6d62..19c3a0223 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MapRegion, MapRegionList, is_64_bit, - buffer, ) import sys @@ -160,7 +159,7 @@ def buffer(self): **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) + return memoryview(self._region.buffer())[self._ofs:self._ofs+self._size] def map(self): """ diff --git a/smmap/util.py b/smmap/util.py index 4e713f8ae..72b2394a3 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -5,22 +5,11 @@ from mmap import mmap, ACCESS_READ from mmap import ALLOCATIONGRANULARITY -__all__ = ["align_to_mmap", "is_64_bit", "buffer", +__all__ = ["align_to_mmap", "is_64_bit", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities -try: - # Python 2 - buffer = buffer -except NameError: - # Python 3 has no `buffer`; only `memoryview` - def buffer(obj, offset, size): - # Actually, for gitpython this is fastest ... . - return memoryview(obj)[offset:offset+size] - # doing it directly is much faster ! - # return obj[offset:offset + size] - def align_to_mmap(num, round_up): """ From 0e59cc906065c36e2605f9b76d797d5227f84460 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:20:13 -0600 Subject: [PATCH 0433/1790] Update Python requirement to >= 3.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f0c67ad2d..c4a9f4e4f 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + python_requires=">=3.4", classifiers=[ # Picked from # http://pypi.python.org/pypi?:action=list_classifiers From ee01c2e6f661b76ddde94cdfa02176dc573cd95b Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:21:16 -0600 Subject: [PATCH 0434/1790] Update setup classifiers to be only Python 3 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c4a9f4e4f..d2c6a21d2 100755 --- a/setup.py +++ b/setup.py @@ -45,12 +45,11 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, long_description_content_type='text/markdown', From 9478708e6663fabbeb8464d6a943054e8927c2ce Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:25:14 -0600 Subject: [PATCH 0435/1790] Update Python prerequisite in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fcb9217f2..9e6ea427c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.7 or 3.4+ +* Python 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. From cc7d2b8516a9bc23d7b3ac599395b80461b738cd Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:26:22 -0600 Subject: [PATCH 0436/1790] Remove information about Python < 2.5 in README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9e6ea427c..9707e5036 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,6 @@ Smmap wraps an interface around mmap and tracks the mapped files as well as the To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. -The interface also works around the missing offset parameter in python implementations up to python 2.5. - Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. From 3646d133e3be66ce1297560542f43b547d47e318 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:27:08 -0600 Subject: [PATCH 0437/1790] Remove Python 2.7 from tox configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d1f558bc0..8a5ce02ad 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py27, py34, py35, py36 +envlist = flake8, py34, py35, py36 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From d20070534a443543fabfae24e249921821a0f991 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:29:39 -0600 Subject: [PATCH 0438/1790] Update Python prerequisite in documentation --- doc/source/intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8f7cf3c26..63480fc28 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.7 or 3.4+ +* Python 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. From 72d47eb937f22de5c71abe5630af6c1dc999bbe0 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:30:19 -0600 Subject: [PATCH 0439/1790] Remove information about Python < 2.5 in documentation --- doc/source/intro.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 63480fc28..a4c82d295 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -13,8 +13,6 @@ Smmap wraps an interface around mmap and tracks the mapped files as well as the To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. -The interface also works around the missing offset parameter in python implementations up to python 2.5. - Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. From 812ad85acd3eda5b1d6dc5aa837d390a72ce1538 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:32:29 -0600 Subject: [PATCH 0440/1790] Replace codecs.open with open --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d2c6a21d2..46eecdd9b 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python import os -import codecs + try: from setuptools import setup, find_packages except ImportError: @@ -11,7 +11,7 @@ import smmap if os.path.exists("README.md"): - long_description = codecs.open('README.md', "r", "utf-8").read().replace('\r\n', '\n') + long_description = open('README.md', "r", encoding="utf-8").read().replace('\r\n', '\n') else: long_description = "See https://github.com/gitpython-developers/smmap" From 960cbc5b01aafe4d0b1706ab236d40d1125e05ac Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:42:41 -0600 Subject: [PATCH 0441/1790] Remove buffer usage from TestTutorial.test_example --- smmap/test/test_tutorial.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index b03db9be2..31c272abb 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -41,12 +41,6 @@ def test_example(self): c.buffer()[1:10] # first 9 bytes c.buffer()[c.size() - 1] # last byte - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - from smmap.util import buffer - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - # you can query absolute offsets, and check whether an offset is included # in the cursor's data. assert c.ofs_begin() < c.ofs_end() From 230725f81e0298d110f8b136e1a6a19c9105d961 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:45:48 -0600 Subject: [PATCH 0442/1790] Remove Development Status classifier comments in setup.py --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index 46eecdd9b..755fddafd 100755 --- a/setup.py +++ b/setup.py @@ -28,15 +28,7 @@ zip_safe=True, python_requires=">=3.4", classifiers=[ - # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", - # "Development Status :: 4 - Beta", "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", From 710fb1994adda7215261d240e6332c2a10bfff6a Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:48:38 -0600 Subject: [PATCH 0443/1790] Remove Travis CI configuration file --- .travis.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 50e5b97e1..000000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -# NOT USED, just for reference. See github actions for CI configuration -language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 -install: - - pip install coveralls -script: - - ulimit -n 48 - - ulimit -n - - nosetests --with-coverage -after_success: - - coveralls From 724540cd82856e850ab9175775db8caa1ea42ec7 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:49:29 -0600 Subject: [PATCH 0444/1790] Remove AppVeyor configuration file --- .appveyor.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 2090a0569..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -# NOT USED, just for reference. See github actions for CI configuration -# CI on Windows via appveyor -environment: - - matrix: - ## MINGW - # - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - - PYTHON: "C:\\Miniconda35-x64" - PYTHON_VERSION: "3.5" - IS_CONDA: "yes" - -install: - - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - ## Print configuration for debugging. - # - - | - echo %PATH% - uname -a - where python pip pip2 pip3 pip34 - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - - - IF "%IS_CONDA%"=="yes" ( - conda info -a & - conda install --yes --quiet pip - ) - - pip install nose wheel coveralls - - ## For commits performed with the default user. - - | - git config --global user.email "travis@ci.com" - git config --global user.name "Travis Runner" - - - pip install -e . - -build: false - -test_script: - - IF "%PYTHON_VERSION%"=="3.5" ( - nosetests -v --with-coverage - ) ELSE ( - nosetests -v - ) From 18b499e213b1fd81c4cf5e5340a5af6302f26bef Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:49:54 -0600 Subject: [PATCH 0445/1790] Remove MemoryManagerError and RegionCollectionError --- smmap/exc.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 smmap/exc.py diff --git a/smmap/exc.py b/smmap/exc.py deleted file mode 100644 index 117664502..000000000 --- a/smmap/exc.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Module with system exceptions""" - - -class MemoryManagerError(Exception): - - """Base class for all exceptions thrown by the memory manager""" - - -class RegionCollectionError(MemoryManagerError): - - """Thrown if a memory region could not be collected, or if no region for collection was found""" From fe64cbd88c37e0a5fb52709790b44cba7a5f5fd8 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:52:16 -0600 Subject: [PATCH 0446/1790] Remove Exceptions documentation --- doc/source/api.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index cddd268c4..2e2dac41a 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -20,14 +20,6 @@ Buffers :members: :undoc-members: -********** -Exceptions -********** - -.. automodule:: smmap.exc - :members: - :undoc-members: - ********* Utilities ********* From 95431a22d559c17e96ed2ca094247f0d227a544c Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:55:43 -0600 Subject: [PATCH 0447/1790] Add Python 3.9 to GitHub Actions workflow --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4f06a18d9..e070c68f8 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 From 8fda48842fb422025b498cd21644b7a480593cc2 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:57:21 -0600 Subject: [PATCH 0448/1790] Add Python 3.7, 3.8, and 3.9 to setup classifiers --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 755fddafd..039f30241 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,9 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, From c64ac58cf768d5416fcc3e306fa72843089313a2 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:00:26 -0600 Subject: [PATCH 0449/1790] Drop support for Python 3.4 --- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9707e5036..412dfd9c0 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.4+ +* Python 3.5+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index a4c82d295..e458d53aa 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.4+ +* Python 3.5+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 039f30241..0718c15ef 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.4", + python_requires=">=3.5", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", From 689ab6fb040e218ca16e4c0aa4944e608649df04 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:02:43 -0600 Subject: [PATCH 0450/1790] Add Python 3.7, 3.8, and 3.9 to tox configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8a5ce02ad..9e8dd22b2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py34, py35, py36 +envlist = flake8, py34, py35, py36, py37, py38, py39 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From ed11471388a3c4dc4756605ec76650e7ad994eb9 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:03:00 -0600 Subject: [PATCH 0451/1790] Remove Python 3.4 from tox configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9e8dd22b2..e33f567f2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py34, py35, py36, py37, py38, py39 +envlist = flake8, py35, py36, py37, py38, py39 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From e762f17a0bc479d52d4f9cda46cec3f98c89e812 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:06:54 -0600 Subject: [PATCH 0452/1790] Remove unused pyvers variable in SlidingWindowMapBuffer.__getslice__ --- smmap/buf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smmap/buf.py b/smmap/buf.py index 18766daf8..af8496eff 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -80,7 +80,6 @@ def __getslice__(self, i, j): ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code - pyvers = sys.version_info[:2] md = list() while l: c.use_region(ofs, l) From 30e93fee57286afae25c28a97ba65a9770f9a729 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:17:37 -0600 Subject: [PATCH 0453/1790] v4.0.0 --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index bc0b1b63c..ff5ca9918 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +****** +v4.0.0 +****** + +- Dropped support for Python 2.7 and 3.4 +- Added support for Python 3.7, 3.8, and 3.9 +- Removed unused exc.MemoryManagerError and exc.RegionCollectionError + ****** v3.0.5 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index abc09d952..45f8abec8 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 5) +version_info = (4, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 447c8a4b0cc6e8fbc4a20a5a6e2c7cfabe05368e Mon Sep 17 00:00:00 2001 From: Tom McClintock Date: Wed, 24 Mar 2021 08:57:44 -0700 Subject: [PATCH 0454/1790] Bumped smmap upper bound --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b6ccf50fa..72957a243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=3.0.1,<4 +smmap>=3.0.1,<5 From aa7228e8dbdc2ee6b6bc385e8bee21245a10e98d Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 25 Mar 2021 06:29:35 -0500 Subject: [PATCH 0455/1790] v4.0.6 --- doc/source/changes.rst | 8 ++++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 05748a08d..6217ffb99 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +***** +4.0.6 +***** + +* Bumped upper bound for smmap requirement + (`#67 `_, + `#68 `_) + ***** 4.0.5 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2e127cac4..ea7d1bcc6 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 5) +version_info = (4, 0, 6) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index facdf3d73..071b8c6c1 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 5) +version_info = (4, 0, 6) __version__ = '.'.join(str(i) for i in version_info) setup( From fc11a03b6d94cdf9d5841595caf104c2982934bb Mon Sep 17 00:00:00 2001 From: Harmon Date: Fri, 26 Mar 2021 07:41:26 -0500 Subject: [PATCH 0456/1790] Update smmap upper bound in setup.py Fixes #69 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 071b8c6c1..edbeeffe3 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap>=3.0.1,<4'], + install_requires=['smmap>=3.0.1,<5'], long_description="""GitDB is a pure-Python git object database""", python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From 6b997cd5fd01dd91ecb08d39e5e9736bc1dc9ba5 Mon Sep 17 00:00:00 2001 From: Harmon Date: Fri, 26 Mar 2021 07:43:45 -0500 Subject: [PATCH 0457/1790] v4.0.7 --- doc/source/changes.rst | 7 +++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6217ffb99..1a41d537b 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +4.0.7 +***** + +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + ***** 4.0.6 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ea7d1bcc6..bef9696c7 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 6) +version_info = (4, 0, 7) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index edbeeffe3..59817d8b2 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 6) +version_info = (4, 0, 7) __version__ = '.'.join(str(i) for i in version_info) setup( From 03ab3a1d40c04d6a944299c21db61cf9ce30f6bb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 25 Mar 2021 20:46:35 +0800 Subject: [PATCH 0458/1790] change signing key to the one I have --- Makefile | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a5098b754..82907561b 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* + twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* doc:: make -C doc/ html diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 5e5f940df..30e93fee5 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 5e5f940dff80beaa3eedf9342ef502f5e630d5ed +Subproject commit 30e93fee57286afae25c28a97ba65a9770f9a729 From dff15cd8ba473776f76e8a3b6359a861e72d74aa Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:09:11 +0300 Subject: [PATCH 0459/1790] Replace deprecated unittest aliases --- gitdb/test/db/lib.py | 4 ++-- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/test_pack.py | 2 +- gitdb/test/test_stream.py | 2 +- gitdb/test/test_util.py | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index c6f4316cd..3df326b68 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -102,8 +102,8 @@ def _assert_object_writing(self, db): assert ostream.type == str_blob_type assert ostream.size == len(data) else: - self.failUnlessRaises(BadObject, db.info, sha) - self.failUnlessRaises(BadObject, db.stream, sha) + self.assertRaises(BadObject, db.info, sha) + self.assertRaises(BadObject, db.stream, sha) # DIRECT STREAM COPY # our data hase been written in object format to the StringIO diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index acc0f153f..6ecf7d7a8 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -43,7 +43,7 @@ def test_reading(self): assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha # END for each sha - self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + self.assertRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 9c25a0249..8cc660b8a 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -32,5 +32,5 @@ def test_basics(self, path): assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha # END for each sha - self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + self.assertRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') # raises if no object could be found diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 9694238bd..458d80434 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -84,4 +84,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) + self.assertRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 8bf78f0d2..48a185290 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -73,7 +73,7 @@ def _assert_index_file(self, index, version, size): assert index.partial_sha_to_index(sha[:l], l * 2) == oidx # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) + self.assertRaises(ValueError, index.partial_sha_to_index, "\0", 2) def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 96268252d..5d4b93db7 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -135,7 +135,7 @@ def test_compressed_writer(self): ostream.close() # its closed already - self.failUnlessRaises(OSError, os.close, fd) + self.assertRaises(OSError, os.close, fd) # read everything back, compare to data we zip fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 847bdab5e..3b3165d14 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -40,8 +40,8 @@ def test_lockedfd(self): lockfilepath = lfd._lockfilepath() # cannot end before it was started - self.failUnlessRaises(AssertionError, lfd.rollback) - self.failUnlessRaises(AssertionError, lfd.commit) + self.assertRaises(AssertionError, lfd.rollback) + self.assertRaises(AssertionError, lfd.commit) # open for writing assert not os.path.isfile(lockfilepath) @@ -77,7 +77,7 @@ def test_lockedfd(self): wfdstream = lfd.open(write=True, stream=True) # this time as stream assert os.path.isfile(lockfilepath) # another one fails - self.failUnlessRaises(IOError, olfd.open) + self.assertRaises(IOError, olfd.open) wfdstream.write(new_data.encode("ascii")) lfd.commit() From 1003c6612e0ee8973ba701e317b7308b7d0136aa Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:22:22 +0300 Subject: [PATCH 0460/1790] Fix Sphinx warnings --- doc/source/conf.py | 2 +- gitdb/db/base.py | 5 +++++ gitdb/db/mem.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 68d9a3fcd..3ab15ab35 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -120,7 +120,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['.static'] +#html_static_path = ['.static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2d7b9fa8d..f0b8a0574 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -33,6 +33,9 @@ def __contains__(self, sha): #{ Query Interface def has_object(self, sha): """ + Whether the object identified by the given 20 bytes + binary sha is contained in the database + :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") @@ -82,6 +85,8 @@ def set_ostream(self, stream): def ostream(self): """ + Return the output stream + :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 5b242e46e..212a68fd1 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -92,6 +92,7 @@ def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly **Note:** the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount of input shas, one or more objects did already exist in odb""" count = 0 From 01b6510de861ed19958ecdd445afaccd2d8a7951 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:31:04 +0300 Subject: [PATCH 0461/1790] Remove redundant Python 2.6 code --- gitdb/stream.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index b94ef245d..d58d1a63e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -30,7 +30,6 @@ from gitdb.utils.encoding import force_bytes has_perf_mod = False -PY26 = sys.version_info[:2] < (2, 7) try: from gitdb_speedups._perf import apply_delta as c_apply_delta has_perf_mod = True @@ -295,7 +294,7 @@ def read(self, size=-1): # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if PY26 or ((zlib.ZLIB_VERSION == '1.2.7' or zlib.ZLIB_VERSION == '1.2.5') and not sys.platform == 'darwin'): + if zlib.ZLIB_VERSION in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) From e4cd296a2101df10e400ec2f1f7ac8b5ac2c37eb Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:32:36 +0300 Subject: [PATCH 0462/1790] Remove redundant Python 2 code --- gitdb/db/mem.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 212a68fd1..b2542ff1b 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -82,10 +82,7 @@ def size(self): return len(self._cache) def sha_iter(self): - try: - return self._cache.iterkeys() - except AttributeError: - return self._cache.keys() + return self._cache.keys() #{ Interface def stream_copy(self, sha_iter, odb): From 0df4b09bac21b7d3e50931c01ccf261419f5e1ef Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:42:16 +0300 Subject: [PATCH 0463/1790] Support Python 3.7-3.9 --- .github/workflows/pythonpackage.yml | 4 ++-- setup.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4c67fbf5b..7f35b616e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/setup.py b/setup.py index 59817d8b2..522a88f16 100755 --- a/setup.py +++ b/setup.py @@ -38,6 +38,8 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7" + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ] ) From 3affd88c09635f0cad0f268c5ca22162c1aa0aa8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:19:42 +0300 Subject: [PATCH 0464/1790] Drop support for EOL Python 3.5 --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e070c68f8..3f1455d96 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9"] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 412dfd9c0..f083dd004 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.5+ +* Python 3.6+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index e458d53aa..3489b04ee 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.5+ +* Python 3.6+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 0718c15ef..833379478 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.5", + python_requires=">=3.6", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/tox.ini b/tox.ini index e33f567f2..54f328da1 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py35, py36, py37, py38, py39 +envlist = flake8, py36, py37, py38, py39 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From 19772d24cd63a0b5392084c07cb6a4e8a2ac86ae Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:06:35 +0300 Subject: [PATCH 0465/1790] Add support for Python 3.10 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3f1455d96..e8726a269 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 833379478..c25e445ed 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, diff --git a/tox.ini b/tox.ini index 54f328da1..6587dcedf 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py36, py37, py38, py39 +envlist = flake8, py36, py37, py38, py39, py310 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From ddc4b9f412769915b77d857324ad165ef488499b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:14:27 +0300 Subject: [PATCH 0466/1790] Switch from abandoned nose to pytest to support Python 3.10 --- .github/workflows/pythonpackage.yml | 8 ++++---- Makefile | 6 +++--- setup.py | 2 -- tox.ini | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e8726a269..e6e68ff7c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -22,7 +22,7 @@ jobs: with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -35,9 +35,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with nose + - name: Test run: | - pip install nose + pip install pytest ulimit -n 48 ulimit -n - nosetests -v + pytest -v diff --git a/Makefile b/Makefile index 593a758ee..9145cabb0 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,11 @@ clean-files: clean: clean-files clean-docs test: - nosetests + pytest coverage: - nosetests --with-coverage --cover-package=smmap - + pytest --cov smmap --cov-report xml + build: ./setup.py build diff --git a/setup.py b/setup.py index c25e445ed..56e560a38 100755 --- a/setup.py +++ b/setup.py @@ -47,6 +47,4 @@ ], long_description=long_description, long_description_content_type='text/markdown', - tests_require=('nose', 'nosexcover'), - test_suite='nose.collector' ) diff --git a/tox.ini b/tox.ini index 6587dcedf..c34ab0290 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,10 @@ envlist = flake8, py36, py37, py38, py39, py310 [testenv] -commands = nosetests {posargs:--with-coverage --cover-package=smmap} +commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} deps = - nose - nosexcover + pytest + pytest-cov [testenv:flake8] commands = flake8 {posargs} From 1bda3b6a5ee6e383e9fd3316caa2d8181e53bf45 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:18:23 +0300 Subject: [PATCH 0467/1790] Universal wheels are for code expected to work on both Python 2 and 3 --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 83a51c4b9..38c535ba5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,2 @@ -[bdist_wheel] -universal = 1 - [flake8] exclude = .tox,.venv,build,dist,doc From f0b322afcf6934501bade7776c5331619485a06c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:27:17 +0300 Subject: [PATCH 0468/1790] Upgrade Python syntax with pyupgrade --py36-plus --- doc/source/conf.py | 9 ++++----- setup.py | 2 +- smmap/buf.py | 2 +- smmap/mman.py | 6 +++--- smmap/test/lib.py | 2 +- smmap/test/test_buf.py | 2 -- smmap/test/test_mman.py | 2 -- smmap/util.py | 6 +++--- 8 files changed, 13 insertions(+), 18 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 5aa28e2a6..55dfc5c46 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # smmap documentation build configuration file, created by # sphinx-quickstart on Wed Jun 8 15:14:25 2011. @@ -38,8 +37,8 @@ master_doc = 'index' # General information about the project. -project = u'smmap' -copyright = u'2011, Sebastian Thiel' +project = 'smmap' +copyright = '2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -173,8 +172,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'smmap.tex', u'smmap Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'smmap.tex', 'smmap Documentation', + 'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/setup.py b/setup.py index 56e560a38..9ab8a7f0c 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ import smmap if os.path.exists("README.md"): - long_description = open('README.md', "r", encoding="utf-8").read().replace('\r\n', '\n') + long_description = open('README.md', encoding="utf-8").read().replace('\r\n', '\n') else: long_description = "See https://github.com/gitpython-developers/smmap" diff --git a/smmap/buf.py b/smmap/buf.py index af8496eff..795e0fd87 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -4,7 +4,7 @@ __all__ = ["SlidingWindowMapBuffer"] -class SlidingWindowMapBuffer(object): +class SlidingWindowMapBuffer: """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. diff --git a/smmap/mman.py b/smmap/mman.py index 19c3a0223..1de7d9e97 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -15,7 +15,7 @@ #}END utilities -class WindowCursor(object): +class WindowCursor: """ Pointer into the mapped region of the memory manager, keeping the map @@ -233,7 +233,7 @@ def fd(self): #} END interface -class StaticWindowMapManager(object): +class StaticWindowMapManager: """Provides a manager which will produce single size cursors that are allowed to always map the whole file. @@ -486,7 +486,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize): """Adjusts the default window size to -1""" - super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + super().__init__(window_size, max_memory_size, max_open_handles) def _obtain_region(self, a, offset, size, flags, is_recursive): # bisect to find an existing region. The c++ implementation cannot diff --git a/smmap/test/lib.py b/smmap/test/lib.py index f86c0c6f1..ca91ee914 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -8,7 +8,7 @@ #{ Utilities -class FileCreator(object): +class FileCreator: """A instance which creates a temporary file with a prefix and a given size and provides this info to the user. diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 3b6009e11..17555afe4 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from .lib import TestBase, FileCreator from smmap.mman import ( diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 96bc355b7..d88316b8e 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from .lib import TestBase, FileCreator from smmap.mman import ( diff --git a/smmap/util.py b/smmap/util.py index 72b2394a3..cf027afdc 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -34,7 +34,7 @@ def is_64_bit(): #{ Utility Classes -class MapWindow(object): +class MapWindow: """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( @@ -80,7 +80,7 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class MapRegion(object): +class MapRegion: """Defines a mapped region of memory, aligned to pagesizes @@ -198,7 +198,7 @@ class MapRegionList(list): ) def __new__(cls, path): - return super(MapRegionList, cls).__new__(cls) + return super().__new__(cls) def __init__(self, path_or_fd): self._path_or_fd = path_or_fd From db8810096503dd8a1f5a021ff39be907417f90a7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2021 21:18:55 +0800 Subject: [PATCH 0469/1790] bump major version and update changelog --- doc/source/changes.rst | 7 +++++++ smmap/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ff5ca9918..d0623b533 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +****** +v5.0.0 +****** + +- Dropped support 3.5 +- Added support for Python 3.10 + ****** v4.0.0 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 45f8abec8..696401c8c 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (4, 0, 0) +version_info = (5, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 47a74ca590efbdc3ae59277a491562efa7acc605 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Oct 2021 08:29:50 +0800 Subject: [PATCH 0470/1790] Allow usage of smmap 5.0 (#76) --- gitdb/ext/smmap | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 30e93fee5..db8810096 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 30e93fee57286afae25c28a97ba65a9770f9a729 +Subproject commit db8810096503dd8a1f5a021ff39be907417f90a7 diff --git a/requirements.txt b/requirements.txt index 72957a243..1b2e11db7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=3.0.1,<5 +smmap>=3.0.1,<6 From 9a289258074fbf92a64186274067a46f7b27666e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Oct 2021 08:36:34 +0800 Subject: [PATCH 0471/1790] Drop support for python 3.4/3.5; make use of smmap 5.0 which does the same --- doc/source/changes.rst | 8 ++++++++ gitdb/__init__.py | 2 +- setup.py | 8 +++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 1a41d537b..de448dd4e 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +***** +4.0.8 +***** + +* drop support for python 3.4 and 3.5 due to EOL +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + ***** 4.0.7 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index bef9696c7..0d936dba5 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 7) +version_info = (4, 0, 8) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 522a88f16..251ea93ba 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 7) +version_info = (4, 0, 8) __version__ = '.'.join(str(i) for i in version_info) setup( @@ -20,9 +20,9 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap>=3.0.1,<5'], + install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.4', + python_requires='>=3.6', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,8 +35,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From 2913a6454c9dfc803679dc5f75315e2d821ee977 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Wed, 5 Jan 2022 21:58:22 +0800 Subject: [PATCH 0472/1790] Fix typos --- gitdb/__init__.py | 2 +- gitdb/db/pack.py | 2 +- gitdb/fun.py | 8 ++++---- gitdb/pack.py | 4 ++-- gitdb/stream.py | 2 +- gitdb/test/db/lib.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/test_pack.py | 2 +- gitdb/util.py | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 0d936dba5..b5d1939dc 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -21,7 +21,7 @@ def _init_externals(): except ImportError as e: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) from e # END verify import - # END handel imports + # END handle imports #} END initialization diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 177ed7bb2..90de02b61 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -131,7 +131,7 @@ def store(self, istream): def update_cache(self, force=False): """ - Update our cache with the acutally existing packs on disk. Add new ones, + Update our cache with the actually existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones :param force: If True, the cache will be updated even though the directory diff --git a/gitdb/fun.py b/gitdb/fun.py index 98465975d..abb4277db 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -103,7 +103,7 @@ def delta_chunk_apply(dc, bbuf, write): write(bbuf[dc.so:dc.so + dc.ts]) else: # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? + # what's faster: if + 4 function calls or just a write with a slice ? # Considering data can be larger than 127 bytes now, it should be worth it if dc.ts < len(dc.data): write(dc.data[:dc.ts]) @@ -292,7 +292,7 @@ def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size :param target_size: if not -1, the total size of the chain must be target_size - :raise AssertionError: if the size doen't match""" + :raise AssertionError: if the size doesn't match""" if target_size > -1: assert self[-1].rbound() == target_size assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size @@ -331,7 +331,7 @@ def connect_with_next_base(self, bdcl): cannot be changed by any of the upcoming bases anymore. Once all our chunks are marked like that, we can stop all processing :param bdcl: data chunk list being one of our bases. They must be fed in - consequtively and in order, towards the earliest ancestor delta + consecutively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams if False is returned""" nfc = 0 # number of frozen chunks @@ -624,7 +624,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf_size: size for the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes diff --git a/gitdb/pack.py b/gitdb/pack.py index a38468e37..0b26c121c 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -224,7 +224,7 @@ def write(self, pack_sha, write): if ofs > 0x7fffffff: tmplist.append(ofs) ofs = 0x80000000 + len(tmplist) - 1 - # END hande 64 bit offsets + # END handle 64 bit offsets sha_write(pack('>L', ofs & 0xffffffff)) # END for each offset @@ -506,7 +506,7 @@ class PackFile(LazyMixin): """A pack is a file written according to the Version 2 for git packs As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + packs therefore is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. **Note:** at some point, this might be implemented using streams as well, or diff --git a/gitdb/stream.py b/gitdb/stream.py index d58d1a63e..37380ad3e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -289,7 +289,7 @@ def read(self, size=-1): # They are thorough, and I assume it is truly working. # Why is this logic as convoluted as it is ? Please look at the table in # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. - # Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch # will be the one that works. # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 3df326b68..b38f1d5e5 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -107,7 +107,7 @@ def _assert_object_writing(self, db): # DIRECT STREAM COPY # our data hase been written in object format to the StringIO - # we pasesd as output stream. No physical database representation + # we passed as output stream. No physical database representation # was created. # Test direct stream copy of object streams, the result must be # identical to what we fed in diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 458d80434..ff96a58ac 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -80,7 +80,7 @@ def test_writing(self, path): # END for each sha to find # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... + # but in our pack, there is no ambiguous ... # assert num_ambiguous # non-existing diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 48a185290..4b01741b2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -101,7 +101,7 @@ def _assert_pack_file(self, pack, version, size): dstream = DeltaApplyReader.new(streams) except ValueError: # ignore these, old git versions use only ref deltas, - # which we havent resolved ( as we are without an index ) + # which we haven't resolved ( as we are without an index ) # Also ignore non-delta streams continue # END get deltastream diff --git a/gitdb/util.py b/gitdb/util.py index c4cafecc6..f9f8c0ed7 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -182,7 +182,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): pass # END exception handling - # read manully + # read manually contents = os.read(fd, os.fstat(fd).st_size) if stream: return _RandomAccessBytesIO(contents) @@ -248,7 +248,7 @@ class LazyMixin(object): def __getattr__(self, attr): """ Whenever an attribute is requested that we do not know, we allow it - to be created and set. Next time the same attribute is reqeusted, it is simply + to be created and set. Next time the same attribute is requested, it is simply returned from our dict/slots. """ self._set_cache_(attr) # will raise in case the cache was not created @@ -332,7 +332,7 @@ def open(self, write=False, stream=False): # open actual file if required if self._fd is None: - # we could specify exlusive here, as we obtained the lock anyway + # we could specify exclusive here, as we obtained the lock anyway try: self._fd = os.open(self._filepath, os.O_RDONLY | binary) except: From 597d36c4ac84af6deb7cecf2a1e8b4ca4b7dccdf Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Sat, 15 Jan 2022 23:28:12 +0800 Subject: [PATCH 0473/1790] Fix typos --- smmap/__init__.py | 2 +- smmap/buf.py | 2 +- smmap/mman.py | 10 +++++----- smmap/test/test_buf.py | 2 +- smmap/test/test_mman.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index 696401c8c..6b1a44cb5 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -1,4 +1,4 @@ -"""Intialize the smmap package""" +"""Initialize the smmap package""" __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" diff --git a/smmap/buf.py b/smmap/buf.py index 795e0fd87..ad27b6974 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -21,7 +21,7 @@ class SlidingWindowMapBuffer: ) def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): - """Initalize the instance to operate on the given cursor. + """Initialize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes diff --git a/smmap/mman.py b/smmap/mman.py index 1de7d9e97..873f687fd 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -27,7 +27,7 @@ class WindowCursor: that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( - '_manager', # the manger keeping all file regions + '_manager', # the manager keeping all file regions '_rlist', # a regions list with regions for our file '_region', # our current class:`MapRegion` or None '_ofs', # relative offset from the actually mapped area to our start area @@ -66,7 +66,7 @@ def _destroy(self): # sometimes, during shutdown, getrefcount is None. Its possible # to re-import it, however, its probably better to just ignore # this python problem (for now). - # The next step is to get rid of the error prone getrefcount alltogether. + # The next step is to get rid of the error prone getrefcount altogether. pass # END exception handling # END handle regions @@ -95,7 +95,7 @@ def __copy__(self): #{ Interface def assign(self, rhs): """Assign rhs to this instance. This is required in order to get a real copy. - Alternativly, you can copy an existing instance using the copy module""" + Alternatively, you can copy an existing instance using the copy module""" self._destroy() self._copy_from(rhs) @@ -342,7 +342,7 @@ def _collect_lru_region(self, size): return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, + """Utility to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" @@ -427,7 +427,7 @@ def mapped_memory_size(self): return self._memory_size def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" + """:return: maximum amount of handles we may have opened""" return self._max_handle_count def max_mapped_memory_size(self): diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 17555afe4..f0a86fb64 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -30,7 +30,7 @@ def test_basics(self): self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - buf = SlidingWindowMapBuffer() # can create uninitailized buffers + buf = SlidingWindowMapBuffer() # can create uninitialized buffers assert buf.cursor() is None # can call end access any time diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index d88316b8e..7a5f4092c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -154,7 +154,7 @@ def test_memman_operation(self): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region() is not rr # old region is still available, but has not curser ref anymore + assert c.region() is not rr # old region is still available, but has not cursor ref anymore assert rr.client_count() == 1 # only held by manager else: assert c.size() < fc.size @@ -194,7 +194,7 @@ def test_memman_operation(self): # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() - # END statics will overshoot, which is fine + # END statistics will overshoot, which is fine assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, (size or c.size())).is_valid() csize = c.size() @@ -205,7 +205,7 @@ def test_memman_operation(self): assert includes_ofs(base_offset + csize - 1) assert not includes_ofs(base_offset + csize) # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000 * 1000) print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), From e3a4cfe0ef2bc7b9c785c4fec650f5045cdd1e50 Mon Sep 17 00:00:00 2001 From: Carl George Date: Wed, 9 Feb 2022 17:15:39 -0600 Subject: [PATCH 0474/1790] Switch from nose to pytest This is not a full rewrite to pytest style tests, it just changes the minimum to allow pytest to run the existing tests. Resolves #72 --- .github/workflows/pythonpackage.yml | 6 +++--- Makefile | 3 +-- README.rst | 4 ++-- gitdb.pro.user | 3 +-- gitdb/test/db/test_pack.py | 4 ++-- gitdb/test/lib.py | 4 ++-- gitdb/test/test_pack.py | 4 ++-- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7f35b616e..c695ad302 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -36,9 +36,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with nose + - name: Test with pytest run: | - pip install nose + pip install pytest ulimit -n 48 ulimit -n - nosetests -v + pytest -v diff --git a/Makefile b/Makefile index 82907561b..e0630283d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ PYTHON = python SETUP = $(PYTHON) setup.py -TESTRUNNER = $(shell which nosetests) TESTFLAGS = all:: @@ -37,5 +36,5 @@ clean:: rm -f *.so coverage:: build - PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=gitdb --with-coverage --cover-erase --cover-inclusive gitdb + PYTHONPATH=. $(PYTHON) -m pytest --cov=gitdb gitdb diff --git a/README.rst b/README.rst index 44b3eddf8..29c70f781 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ If you want to go up to 20% faster, you can install gitdb-speedups with: REQUIREMENTS ============ -* Python Nose - for running the tests +* pytest - for running the tests SOURCE ====== @@ -45,7 +45,7 @@ Once the clone is complete, please be sure to initialize the submodules using Run the tests with - nosetests + pytest DEVELOPMENT =========== diff --git a/gitdb.pro.user b/gitdb.pro.user index 398cb70a1..3ca1e2182 100644 --- a/gitdb.pro.user +++ b/gitdb.pro.user @@ -233,8 +233,7 @@ - /usr/bin/nosetests - -s + /usr/bin/pytest gitdb/test/test_pack.py 2 diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index ff96a58ac..4539f4278 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -16,7 +16,7 @@ import random import sys -from nose.plugins.skip import SkipTest +import pytest class TestPackDB(TestDBBase): @@ -24,7 +24,7 @@ class TestPackDB(TestDBBase): @with_packs_rw def test_writing(self, path): if sys.platform == "win32": - raise SkipTest("FIXME: Currently fail on windows") + pytest.skip("FIXME: Currently fail on windows") pdb = PackedDB(path) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index a04084f5f..abd4ad57d 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -65,8 +65,8 @@ def skip_on_travis_ci(func): @wraps(func) def wrapper(self, *args, **kwargs): if 'TRAVIS' in os.environ: - import nose - raise nose.SkipTest("Cannot run on travis-ci") + import pytest + pytest.skip("Cannot run on travis-ci") # end check for travis ci return func(self, *args, **kwargs) # end wrapper diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 4b01741b2..f9461975b 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -26,7 +26,7 @@ from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from nose import SkipTest +import pytest import os import tempfile @@ -246,4 +246,4 @@ def rewind_streams(): def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack - raise SkipTest() + pytest.skip('not implemented') From 7c67010acf541b4269825cb2c13e226fe2d65ea9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 24 Oct 2021 21:38:51 +0800 Subject: [PATCH 0475/1790] bump patch level in the hopes for a valid sig on release https://github.com/gitpython-developers/gitdb/issues/77 --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index de448dd4e..5ef1326d0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +4.0.9 +***** + +- re-release of 4.0.8 to get a valid signature. + ***** 4.0.8 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index b5d1939dc..246014554 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 8) +version_info = (4, 0, 9) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 251ea93ba..31dc62f47 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 8) +version_info = (4, 0, 9) __version__ = '.'.join(str(i) for i in version_info) setup( From 4762d99d978586fcdf08ade552f4712bfde6ef22 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 21 Feb 2022 10:43:54 +0800 Subject: [PATCH 0476/1790] Stop signing releases (#77) The key stopped producing correct signatures when upgrading MacOS, at least when used by `twine`. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e0630283d..a7acbb10e 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* + twine upload dist/* doc:: make -C doc/ html From 334ef84a05c953ed5dbec7b9c6d4310879eeab5a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 21 Feb 2022 10:45:33 +0800 Subject: [PATCH 0477/1790] Stop signing releases Related to https://github.com/gitpython-developers/gitdb/issues/77 --- Makefile | 2 +- smmap/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9145cabb0..051e99ba0 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,4 @@ release: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* + twine upload dist/* diff --git a/smmap/util.py b/smmap/util.py index cf027afdc..fbb387211 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -71,7 +71,7 @@ def extend_left_to(self, window, max_size): rofs = self.ofs - window.ofs_end() nsize = rofs + self.size rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs + self.ofs -= rofs self.size += rofs def extend_right_to(self, window, max_size): From 2ce0e3175bcbbf397d25f18b1008d1fdf54611f2 Mon Sep 17 00:00:00 2001 From: randymcmillan Date: Tue, 15 Mar 2022 17:51:07 -0400 Subject: [PATCH 0478/1790] .gitmodules: update smmap url Github.com is currenly redirecting the smmap url: https://github.com/Byron/smmap.git to: https://github.com/gitpython-developers/smmap For correctness we update to the url directly --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index d85b15c9f..e73cdedab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "smmap"] path = gitdb/ext/smmap - url = https://github.com/Byron/smmap.git + url = https://github.com/gitpython-developers/smmap.git From a8cdf46b26f2f7029d3749bc431e2b80fe28b04a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:04:02 +0200 Subject: [PATCH 0479/1790] Add support for Python 3.11 --- .github/workflows/pythonpackage.yml | 6 +++--- tox.ini | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e6e68ff7c..d593fd066 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/tox.ini b/tox.ini index c34ab0290..092c79daa 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py36, py37, py38, py39, py310 +envlist = flake8, py{36, 37, 38, 39, 310, 311} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 0370014ee74a0f4480127932a78f177f6f433b9e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:06:32 +0200 Subject: [PATCH 0480/1790] Drop support for EOL Python 3.6 --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- smmap/buf.py | 2 +- tox.ini | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d593fd066..9e2d881de 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index f083dd004..e21f53453 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.6+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 3489b04ee..109fec247 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.6+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 9ab8a7f0c..1df3b62aa 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.6", + python_requires=">=3.7", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/smmap/buf.py b/smmap/buf.py index ad27b6974..731b0644b 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -93,7 +93,7 @@ def __getslice__(self, i, j): d = d.tobytes() md.append(d) # END while there are bytes to read - return bytes().join(md) + return b''.join(md) # END fast or slow path #{ Interface diff --git a/tox.ini b/tox.ini index 092c79daa..810baddc3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{36, 37, 38, 39, 310, 311} +envlist = flake8, py{37, 38, 39, 310, 311} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 5f149c1a9c36c985158c0757377af958362b3582 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:13:01 +0200 Subject: [PATCH 0481/1790] Add support for Python 3.10 and 3.11 --- .github/workflows/pythonpackage.yml | 6 +++--- setup.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c695ad302..3c05764f5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/setup.py b/setup.py index 31dc62f47..5d324cd96 100755 --- a/setup.py +++ b/setup.py @@ -39,5 +39,8 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", ] ) From a6f0856d807efc1e7bc37d13f9cfbcdb91dea2ac Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:15:02 +0200 Subject: [PATCH 0482/1790] Remove redundant Travis CI config/code --- .travis.yml | 20 ------------------- gitdb/test/lib.py | 15 -------------- gitdb/test/performance/test_pack.py | 4 ---- gitdb/test/performance/test_pack_streaming.py | 3 --- gitdb/test/performance/test_stream.py | 2 -- 5 files changed, 44 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b980d36e9..000000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -# NOT USED, just for reference. See github actions for CI configuration -language: python -python: - - "3.4" - - "3.5" - - "3.6" - # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) - -git: - # a higher depth is needed for one of the tests - lets fet - depth: 1000 -install: - - pip install coveralls -script: - - ulimit -n 48 - - ulimit -n - - nosetests -v --with-coverage -after_success: - - coveralls - diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index abd4ad57d..465a899a9 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -58,21 +58,6 @@ def setUpClass(cls): #{ Decorators -def skip_on_travis_ci(func): - """All tests decorated with this one will raise SkipTest when run on travis ci. - Use it to workaround difficult to solve issues - NOTE: copied from bcore (https://github.com/Byron/bcore)""" - @wraps(func) - def wrapper(self, *args, **kwargs): - if 'TRAVIS' in os.environ: - import pytest - pytest.skip("Cannot run on travis-ci") - # end check for travis ci - return func(self, *args, **kwargs) - # end wrapper - return wrapper - - def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test succeeds, but leave it otherwise to aid additional debugging""" diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b59d5a971..643186b06 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -17,7 +17,6 @@ from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB -from gitdb.test.lib import skip_on_travis_ci import sys import os @@ -26,7 +25,6 @@ class TestPackedDBPerformance(TestBigRepoR): - @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -79,7 +77,6 @@ def test_pack_random_access(self): print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) - @skip_on_travis_ci def test_loose_correctness(self): """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back into the loose object db (memory). @@ -106,7 +103,6 @@ def test_loose_correctness(self): mdb._cache.clear() # end for each sha to copy - @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 76f0f4a07..5bf67909b 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -12,7 +12,6 @@ from gitdb.db.pack import PackedDB from gitdb.stream import NullStream from gitdb.pack import PackEntity -from gitdb.test.lib import skip_on_travis_ci import os import sys @@ -34,7 +33,6 @@ def write(self, d): class TestPackStreamingPerformance(TestBigRepoR): - @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well @@ -61,7 +59,6 @@ def test_pack_writing(self): print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) - @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 92d28e4a3..9a8b15b2d 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -20,7 +20,6 @@ from gitdb.test.lib import ( make_memory_file, with_rw_directory, - skip_on_travis_ci ) @@ -44,7 +43,6 @@ class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB - @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) From faed217d14965932b333c07219ed401a661d2651 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:16:09 +0200 Subject: [PATCH 0483/1790] Drop support for EOL Python 3.5 and 3.6 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3c05764f5..0d039adae 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/setup.py b/setup.py index 5d324cd96..d38b267ce 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.6', + python_requires='>=3.7', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,7 +35,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", From 7a68270d6c78b81f577b433dc6351b26bc27b7cf Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:17:20 +0200 Subject: [PATCH 0484/1790] Upgrade Python syntax with pyupgrade --py37-plus --- doc/source/conf.py | 9 ++++----- gitdb/db/base.py | 12 ++++++------ gitdb/db/git.py | 4 ++-- gitdb/db/loose.py | 4 ++-- gitdb/db/mem.py | 2 +- gitdb/db/pack.py | 2 +- gitdb/db/ref.py | 8 ++++---- gitdb/fun.py | 2 +- gitdb/pack.py | 4 ++-- gitdb/stream.py | 12 ++++++------ gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 6 +++--- gitdb/test/performance/test_pack.py | 1 - gitdb/test/performance/test_pack_streaming.py | 1 - gitdb/test/performance/test_stream.py | 1 - gitdb/test/test_example.py | 2 +- gitdb/test/test_stream.py | 4 ++-- gitdb/util.py | 14 +++++++------- 18 files changed, 43 insertions(+), 47 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 3ab15ab35..b387f6094 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # GitDB documentation build configuration file, created by # sphinx-quickstart on Wed Jun 30 00:01:32 2010. @@ -38,8 +37,8 @@ master_doc = 'index' # General information about the project. -project = u'GitDB' -copyright = u'2011, Sebastian Thiel' +project = 'GitDB' +copyright = '2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -172,8 +171,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'GitDB.tex', u'GitDB Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'GitDB.tex', 'GitDB Documentation', + 'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/gitdb/db/base.py b/gitdb/db/base.py index f0b8a0574..e89052e88 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,7 +22,7 @@ __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') -class ObjectDBR(object): +class ObjectDBR: """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" @@ -63,7 +63,7 @@ def sha_iter(self): #} END query interface -class ObjectDBW(object): +class ObjectDBW: """Defines an interface to create objects in the database""" @@ -105,7 +105,7 @@ def store(self, istream): #} END edit interface -class FileDBBase(object): +class FileDBBase: """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" @@ -117,7 +117,7 @@ def __init__(self, root_path): **Note:** The base will not perform any accessablity checking as the base might not yet be accessible, but become accessible before the first access.""" - super(FileDBBase, self).__init__() + super().__init__() self._root_path = root_path #{ Interface @@ -133,7 +133,7 @@ def db_path(self, rela_path): #} END interface -class CachingDB(object): +class CachingDB: """A database which uses caches to speed-up access""" @@ -176,7 +176,7 @@ def _set_cache_(self, attr): elif attr == '_db_cache': self._db_cache = dict() else: - super(CompoundDB, self)._set_cache_(attr) + super()._set_cache_(attr) def _db_query(self, sha): """:return: database containing the given 20 byte sha diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 7a43d7235..e2cb46805 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -39,7 +39,7 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): def __init__(self, root_path): """Initialize ourselves on a git objects directory""" - super(GitDB, self).__init__(root_path) + super().__init__(root_path) def _set_cache_(self, attr): if attr == '_dbs' or attr == '_loose_db': @@ -68,7 +68,7 @@ def _set_cache_(self, attr): # finally set the value self._loose_db = loose_db else: - super(GitDB, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs #{ ObjectDBW interface diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index a63a2ef33..4ef775049 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -75,7 +75,7 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): new_objects_mode = int("644", 8) def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) + super().__init__(root_path) self._hexsha_to_file = dict() # Additional Flags - might be set to 0 after the first failure # Depending on the root, this might work for some mounts, for others not, which @@ -151,7 +151,7 @@ def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" if stream is not None and not isinstance(stream, Sha1Writer): raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) + return super().set_ostream(stream) def info(self, sha): m = self._map_loose_object(sha) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index b2542ff1b..1b954cba2 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -37,7 +37,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): exists in the target storage before introducing actual IO""" def __init__(self): - super(MemoryDB, self).__init__() + super().__init__() self._db = LooseObjectDB("path/doesnt/matter") # maps 20 byte shas to their OStream objects diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 90de02b61..1ce786b79 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -39,7 +39,7 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): _sort_interval = 500 def __init__(self, root_path): - super(PackedDB, self).__init__(root_path) + super().__init__(root_path) # list of lists with three items: # * hits - number of times the pack was hit with a request # * entity - Pack entity instance diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 2bb1de790..6bb2a64dd 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -20,7 +20,7 @@ class ReferenceDB(CompoundDB): ObjectDBCls = None def __init__(self, ref_file): - super(ReferenceDB, self).__init__() + super().__init__() self._ref_file = ref_file def _set_cache_(self, attr): @@ -28,7 +28,7 @@ def _set_cache_(self, attr): self._dbs = list() self._update_dbs_from_ref_file() else: - super(ReferenceDB, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs def _update_dbs_from_ref_file(self): @@ -44,7 +44,7 @@ def _update_dbs_from_ref_file(self): try: with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] - except (OSError, IOError): + except OSError: pass # END handle alternates @@ -79,4 +79,4 @@ def _update_dbs_from_ref_file(self): def update_cache(self, force=False): # re-read alternates and update databases self._update_dbs_from_ref_file() - return super(ReferenceDB, self).update_cache(force) + return super().update_cache(force) diff --git a/gitdb/fun.py b/gitdb/fun.py index abb4277db..a4454de4d 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -113,7 +113,7 @@ def delta_chunk_apply(dc, bbuf, write): # END handle chunk mode -class DeltaChunk(object): +class DeltaChunk: """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" diff --git a/gitdb/pack.py b/gitdb/pack.py index 0b26c121c..cabce4c9d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -170,7 +170,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): #} END utilities -class IndexWriter(object): +class IndexWriter: """Utility to cache index information, allowing to write all information later in one go to the given stream @@ -257,7 +257,7 @@ class PackIndexFile(LazyMixin): index_version_default = 2 def __init__(self, indexpath): - super(PackIndexFile, self).__init__() + super().__init__() self._indexpath = indexpath def close(self): diff --git a/gitdb/stream.py b/gitdb/stream.py index 37380ad3e..222b843e9 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -219,13 +219,13 @@ def read(self, size=-1): # END clamp size if size == 0: - return bytes() + return b'' # END handle depletion # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream - dat = bytes() + dat = b'' if self._buf: if self._buflen >= size: # have enough data @@ -553,7 +553,7 @@ def size(self): #{ W Streams -class Sha1Writer(object): +class Sha1Writer: """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" @@ -650,7 +650,7 @@ class FDCompressedSha1Writer(Sha1Writer): exc = IOError("Failed to write all bytes to filedescriptor") def __init__(self, fd): - super(FDCompressedSha1Writer, self).__init__() + super().__init__() self.fd = fd self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) @@ -677,7 +677,7 @@ def close(self): #} END stream interface -class FDStream(object): +class FDStream: """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream @@ -711,7 +711,7 @@ def close(self): close(self._fd) -class NullStream(object): +class NullStream: """A stream that does nothing but providing a stream interface. Use it like /dev/null""" diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 20496985e..664aa54ba 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -23,7 +23,7 @@ def make_alt_file(self, alt_path, alt_list): The list can be empty""" with open(alt_path, "wb") as alt_file: for alt in alt_list: - alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) + alt_file.write(alt.encode("utf-8") + b"\n") @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 465a899a9..da59d3b35 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -40,7 +40,7 @@ class TestBase(unittest.TestCase): @classmethod def setUpClass(cls): try: - super(TestBase, cls).setUpClass() + super().setUpClass() except AttributeError: pass @@ -70,7 +70,7 @@ def wrapper(self): try: return func(self, path) except Exception: - sys.stderr.write("Test {}.{} failed, output is at {!r}\n".format(type(self).__name__, func.__name__, path)) + sys.stderr.write(f"Test {type(self).__name__}.{func.__name__} failed, output is at {path!r}\n") keep = True raise finally: @@ -161,7 +161,7 @@ def make_memory_file(size_in_bytes, randomize=False): #{ Stream Utilities -class DummyStream(object): +class DummyStream: def __init__(self): self.was_read = False diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 643186b06..f034baffd 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -3,7 +3,6 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" -from __future__ import print_function from gitdb.test.performance.lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 5bf67909b..db790f1db 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -3,7 +3,6 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" -from __future__ import print_function from gitdb.test.performance.lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 9a8b15b2d..91dc89178 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -3,7 +3,6 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" -from __future__ import print_function from gitdb.test.performance.lib import TestBigRepoR from gitdb.db import LooseObjectDB diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 6e80bf5c6..cc4d40dd3 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -32,7 +32,7 @@ def test_base(self): pass # END ignore exception if there are no loose objects - data = "my data".encode("ascii") + data = b"my data" istream = IStream("blob", len(data), BytesIO(data)) # the object does not yet have a sha diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 5d4b93db7..5e2e1bab2 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -115,13 +115,13 @@ def test_decompress_reader(self): def test_sha_writer(self): writer = Sha1Writer() - assert 2 == writer.write("hi".encode("ascii")) + assert 2 == writer.write(b"hi") assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 # make sure it does something ;) prev_sha = writer.sha() - writer.write("hi again".encode("ascii")) + writer.write(b"hi again") assert writer.sha() != prev_sha def test_compressed_writer(self): diff --git a/gitdb/util.py b/gitdb/util.py index f9f8c0ed7..3151c061e 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -94,7 +94,7 @@ def remove(*args, **kwargs): #{ compatibility stuff ... -class _RandomAccessBytesIO(object): +class _RandomAccessBytesIO: """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" @@ -131,7 +131,7 @@ def byte_ord(b): #{ Routines -def make_sha(source=''.encode("ascii")): +def make_sha(source=b''): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ @@ -151,7 +151,7 @@ def allocate_memory(size): try: return mmap.mmap(-1, size) # read-write by default - except EnvironmentError: + except OSError: # setup real memory instead # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on @@ -174,7 +174,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # supports stream and random access try: return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - except EnvironmentError: + except OSError: # python 2.4 issue, 0 wants to be the actual size return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) # END handle python 2.4 @@ -234,7 +234,7 @@ def to_bin_sha(sha): #{ Utilities -class LazyMixin(object): +class LazyMixin: """ Base class providing an interface to lazily retrieve attribute values upon @@ -266,7 +266,7 @@ def _set_cache_(self, attr): pass -class LockedFD(object): +class LockedFD: """ This class facilitates a safe read and write operation to a file on disk. @@ -327,7 +327,7 @@ def open(self, write=False, stream=False): self._fd = fd # END handle file descriptor except OSError as e: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) from e + raise OSError("Lock at %r could not be obtained" % self._lockfilepath()) from e # END handle lock retrieval # open actual file if required From c3ab5d7b28062848c2a639a60e0acfbaee7e8f90 Mon Sep 17 00:00:00 2001 From: zwimer Date: Tue, 22 Nov 2022 18:57:47 -0700 Subject: [PATCH 0485/1790] Prefer import to __import__ --- .gitignore | 1 + gitdb/__init__.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index e0b4e8579..8b7da9277 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ *.so .noseids *.sublime-workspace +*.egg-info diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 246014554..c5b554746 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -12,15 +12,12 @@ def _init_externals(): """Initialize external projects by putting them into the path""" - for module in ('smmap',): - if 'PYOXIDIZER' not in os.environ: - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - - try: - __import__(module) - except ImportError as e: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) from e - # END verify import + if 'PYOXIDIZER' not in os.environ: + where = os.path.join(os.path.dirname(__file__), 'ext', 'smmap') + if os.path.exists(where): + sys.path.append(where) + + import smmap # END handle imports #} END initialization From 1edc7d296af635dc31030a09e73fd684eedc1d59 Mon Sep 17 00:00:00 2001 From: zwimer Date: Tue, 22 Nov 2022 19:05:11 -0700 Subject: [PATCH 0486/1790] Add del smmap to match previous behavior --- gitdb/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c5b554746..94b0831ac 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -18,6 +18,7 @@ def _init_externals(): sys.path.append(where) import smmap + del smmap # END handle imports #} END initialization From 38c68d95eaed9bf7415781ca0102441a349893ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Nov 2022 06:22:21 +0100 Subject: [PATCH 0487/1790] bump version to 4.0.10 --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 5ef1326d0..7d85ea55b 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +4.0.10 +****** + +- improvements to the way external packages are imported. + ***** 4.0.9 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 94b0831ac..b777632bb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -28,7 +28,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 9) +version_info = (4, 0, 10) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index db8810096..334ef84a0 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit db8810096503dd8a1f5a021ff39be907417f90a7 +Subproject commit 334ef84a05c953ed5dbec7b9c6d4310879eeab5a diff --git a/setup.py b/setup.py index d38b267ce..521484978 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 9) +version_info = (4, 0, 10) __version__ = '.'.join(str(i) for i in version_info) setup( From 49c3178711ddb3510f0e96297187f823cc019871 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Nov 2022 06:24:37 +0100 Subject: [PATCH 0488/1790] use python3 binary MacOS Ventura doesn't come with a `python` binary anymore --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a7acbb10e..7aa5a717a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON = python +PYTHON = python3 SETUP = $(PYTHON) setup.py TESTFLAGS = From b6faecc46fcc4f6412149f2021447c0070eba60e Mon Sep 17 00:00:00 2001 From: Ondrej Tethal Date: Fri, 9 Dec 2022 10:21:58 +0100 Subject: [PATCH 0489/1790] Use ZLIB_RUNTIME_VERSION if available --- gitdb/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 222b843e9..1b5426f02 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -294,7 +294,7 @@ def read(self, size=-1): # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if zlib.ZLIB_VERSION in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) From 97cdb403fd11778916b006b22679f427a3c1a8ac Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sat, 8 Jul 2023 15:22:33 -0400 Subject: [PATCH 0490/1790] Made the init repo section of quickdoc --- doc/source/quickstart.rst | 37 +++++++++++++++++++++++++++++++++++++ test/test_quick_doc.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 doc/source/quickstart.rst create mode 100644 test/test_quick_doc.py diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst new file mode 100644 index 000000000..602ca423f --- /dev/null +++ b/doc/source/quickstart.rst @@ -0,0 +1,37 @@ +.. _quickdoc_toplevel: + +.. highlight:: python + +.. _quickdoc-label: + +============================== +GitPython Quick Start Tutorial +============================== + +git.Repo +******** + +There are a few ways to create a :class:`git.Repo ` object + +An existing local path +###################### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [1-test_init_repo_object] + :end-before: # ![1-test_init_repo_object] + +Existing local git Repo +####################### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [2-test_init_repo_object] + :end-before: # ![2-test_init_repo_object] + +Clone from URL +############## + +For the rest of this tutorial we will use a clone from https://github.com \ No newline at end of file diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py new file mode 100644 index 000000000..a6dfb8e53 --- /dev/null +++ b/test/test_quick_doc.py @@ -0,0 +1,33 @@ +import pytest + +import git +from test.lib import TestBase +from test.lib.helper import with_rw_directory + + +class QuickDoc(TestBase): + def tearDown(self): + import gc + + gc.collect() + + @with_rw_directory + def test_init_repo_object(self, rw_dir): + path_to_dir = rw_dir + + # [1-test_init_repo_object] + from git import Repo + + repo = Repo.init(path_to_dir) + assert repo.__class__ is Repo # Test to confirm repo was initialized + # ![1-test_init_repo_object] + + # [2-test_init_repo_object] + try: + repo = Repo(path_to_dir) + except git.NoSuchPathError: + assert False, f"No such path {path_to_dir}" + # ! [2-test_init_repo_object] + + # [3 - test_init_repo_object] + From 6a9154b1bfcebe7ee28edebec6617993ad6a5569 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sat, 8 Jul 2023 20:52:04 -0400 Subject: [PATCH 0491/1790] Added git clone & git add --- doc/source/quickstart.rst | 27 +++++++++++++++++++++++++- test/test_quick_doc.py | 40 ++++++++++++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 602ca423f..ebdb2520a 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -16,6 +16,8 @@ There are a few ways to create a :class:`git.Repo ` object An existing local path ###################### +$ git init path/to/dir + .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 @@ -34,4 +36,27 @@ Existing local git Repo Clone from URL ############## -For the rest of this tutorial we will use a clone from https://github.com \ No newline at end of file +For the rest of this tutorial we will use a clone from https://github.com/LeoDaCoda/GitPython-TestFileSys.git + +git clone https://some_repo_url + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [1-test_cloned_repo_object] + :end-before: # ![1-test_cloned_repo_object] + +Usage +**************** + +* git add filepath + + + + +* git commit -m message +* git log file +* git status + + + diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index a6dfb8e53..0188367cf 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -1,6 +1,6 @@ import pytest -import git + from test.lib import TestBase from test.lib.helper import with_rw_directory @@ -18,16 +18,46 @@ def test_init_repo_object(self, rw_dir): # [1-test_init_repo_object] from git import Repo - repo = Repo.init(path_to_dir) - assert repo.__class__ is Repo # Test to confirm repo was initialized + repo = Repo.init(path_to_dir) # git init path/to/dir + assert repo.__class__ is Repo # Test to confirm repo was initialized # ![1-test_init_repo_object] # [2-test_init_repo_object] + import git + try: repo = Repo(path_to_dir) except git.NoSuchPathError: assert False, f"No such path {path_to_dir}" - # ! [2-test_init_repo_object] + # ![2-test_init_repo_object] + + @with_rw_directory + def test_cloned_repo_object(self, rw_dir): + local_dir = rw_dir - # [3 - test_init_repo_object] + from git import Repo + import git + # code to clone from url + # [1-test_cloned_repo_object] + repo_url = "https://github.com/LeoDaCoda/GitPython-TestFileSys.git" + + try: + repo = Repo.clone_from(repo_url, local_dir) + except git.CommandError: + assert False, f"Invalid address {repo_url}" + # ![1-test_cloned_repo_object] + + # code to add files + # [2-test_cloned_repo_object] + # We must make a change to a file so that we can add the update to git + + update_file = 'dir1/file2.txt' # we'll use /dir1/file2.txt + with open(f"{local_dir}/{update_file}", 'a') as f: + f.write('\nUpdate version 2') + # ![2-test_cloned_repo_object] + + # [3-test_cloned_repo_object] + add_file = [f"{local_dir}/{update_file}"] + repo.index.add(add_file) # notice the add function requires a list of paths + # [3-test_cloned_repo_object] From 3c42baebf5bd7c509b9962d1490f59e8874f1323 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 9 Jul 2023 05:34:09 -0400 Subject: [PATCH 0492/1790] Finishing touches for Repo quickstart --- doc/source/quickstart.rst | 85 +++++++++++++++++++++++++++++++++++++-- test/test_quick_doc.py | 71 ++++++++++++++++++++++++++++++-- 2 files changed, 149 insertions(+), 7 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index ebdb2520a..0a728e485 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -49,14 +49,91 @@ git clone https://some_repo_url Usage **************** -* git add filepath +* $ git add filepath +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [2-test_cloned_repo_object] + :end-before: # ![2-test_cloned_repo_object] + +Now lets add the updated file to git + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [3-test_cloned_repo_object] + :end-before: # ![3-test_cloned_repo_object] + +Notice the add method requires a list as a parameter + +* $ git commit -m message + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [4-test_cloned_repo_object] + :end-before: # ![4-test_cloned_repo_object] + +* $ git log file + +A list of commits associated with a file + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [5-test_cloned_repo_object] + :end-before: # ![5-test_cloned_repo_object] + +Notice this returns a generator object + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [6-test_cloned_repo_object] + :end-before: # ![6-test_cloned_repo_object] + +returns list of :class:`Commit ` objects + +* $ git status + + * Untracked files + + Lets create a new file + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [7-test_cloned_repo_object] + :end-before: # ![7-test_cloned_repo_object] + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [8-test_cloned_repo_object] + :end-before: # ![8-test_cloned_repo_object] + + * Modified files + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [9-test_cloned_repo_object] + :end-before: # ![9-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [10-test_cloned_repo_object] + :end-before: # ![10-test_cloned_repo_object] + returns a list of :class:`Diff ` objects -* git commit -m message -* git log file -* git status + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [11-test_cloned_repo_object] + :end-before: # ![11-test_cloned_repo_object] diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 0188367cf..bb3372905 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -51,13 +51,78 @@ def test_cloned_repo_object(self, rw_dir): # [2-test_cloned_repo_object] # We must make a change to a file so that we can add the update to git - update_file = 'dir1/file2.txt' # we'll use /dir1/file2.txt + update_file = 'dir1/file2.txt' # we'll use ./dir1/file2.txt with open(f"{local_dir}/{update_file}", 'a') as f: f.write('\nUpdate version 2') # ![2-test_cloned_repo_object] # [3-test_cloned_repo_object] - add_file = [f"{local_dir}/{update_file}"] + add_file = [f"{update_file}"] # relative path from git root repo.index.add(add_file) # notice the add function requires a list of paths - # [3-test_cloned_repo_object] + # ![3-test_cloned_repo_object] + + # code to commit - not sure how to test this + # [4-test_cloned_repo_object] + repo.index.commit("Update to file2") + # ![4-test_cloned_repo_object] + + # [5-test_cloned_repo_object] + file = 'dir1/file2.txt' # relative path from git root + repo.iter_commits('--all', max_count=100, paths=file) + + # Outputs: + + # ![5-test_cloned_repo_object] + + # [6-test_cloned_repo_object] + commits_for_file_generator = repo.iter_commits('--all', max_count=100, paths=file) + commits_for_file = [c for c in commits_for_file_generator] + commits_for_file + + # Outputs: [, + # ] + # ![6-test_cloned_repo_object] + + # Untracked files - create new file + # [7-test_cloned_repo_object] + # We'll create a file5.txt + + file5 = f'{local_dir}/file5.txt' + with open(file5, 'w') as f: + f.write('file5 version 1') + # ![7-test_cloned_repo_object] + + # [8-test_cloned_repo_object] + repo.untracked_files + # Output: ['file5.txt'] + # ![8-test_cloned_repo_object] + + # Modified files + # [9-test_cloned_repo_object] + # Lets modify one of our tracked files + file3 = f'{local_dir}/Downloads/file3.txt' + with open(file3, 'w') as f: + f.write('file3 version 2') # overwrite file 3 + # ![9-test_cloned_repo_object] + + # [10-test_cloned_repo_object] + repo.index.diff(None) + # Output: [, + # ] + # ![10-test_cloned_repo_object] + + # [11-test_cloned_repo_object] + diffs = repo.index.diff(None) + for d in diffs: + print(d.a_path) + + # Downloads/file3.txt + # file4.txt + # ![11-test_cloned_repo_object] + + + + + + From 10ea113ca6141b8a74e78858e6ff6f52bfd04d9f Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 9 Jul 2023 21:29:26 -0400 Subject: [PATCH 0493/1790] finished code for quickstart --- doc/source/quickstart.rst | 70 +++++++++++++++++++++++++++++++++++++++ test/test_quick_doc.py | 46 +++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 0a728e485..9d63c5674 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -136,4 +136,74 @@ returns list of :class:`Commit ` objects :end-before: # ![11-test_cloned_repo_object] +Trees & Blobs +************** +Latest Commit Tree +################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [12-test_cloned_repo_object] + :end-before: # ![12-test_cloned_repo_object] + +Any Commit Tree +############### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [13-test_cloned_repo_object] + :end-before: # ![13-test_cloned_repo_object] + +Display level 1 Contents +######################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [14-test_cloned_repo_object] + :end-before: # ![14-test_cloned_repo_object] + +Recurse through the Tree +######################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [15-test_cloned_repo_object] + :end-before: # ![15-test_cloned_repo_object] + +.. code-block:: python + + print_files_from_git(tree) + +.. code-block:: python + + # Output + | Downloads, tree + ----| Downloads/file3.txt, blob + | dir1, tree + ----| dir1/file1.txt, blob + ----| dir1/file2.txt, blob + | file4.txt, blob + + +Print file version +################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [16-test_cloned_repo_object] + :end-before: # ![16-test_cloned_repo_object] + +.. code-block:: python + + blob = tree[print_file] + print(blob.data_stream.read().decode()) + + # Output + # file 2 version 1 + # Update version 2 diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index bb3372905..2a95bfff5 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -120,7 +120,53 @@ def test_cloned_repo_object(self, rw_dir): # file4.txt # ![11-test_cloned_repo_object] + '''Trees and Blobs''' + # Latest commit tree + # [12-test_cloned_repo_object] + tree = repo.tree() + # ![12-test_cloned_repo_object] + + # Previous commit tree + # [13-test_cloned_repo_object] + prev_commits = [c for c in repo.iter_commits('--all', max_count=10)] + tree = prev_commits[0].tree + # ![13-test_cloned_repo_object] + + # Iterating through tree + # [14-test_cloned_repo_object] + tree = repo.tree() + files_dirs = [fd for fd in tree] + files_dirs + + # Output + # [, + # , + # ] + + # ![14-test_cloned_repo_object] + + # [15-test_cloned_repo_object] + def print_files_from_git(tree, delim='-', i=0): + files_dirs = [fd for fd in tree] + for fd in files_dirs: + print(f'{delim if i != 0 else ""}| {fd.path}, {fd.type}') + if fd.type == "tree": + print_files_from_git(fd, delim * 4, i + 1) + + # ![15-test_cloned_repo_object] + + # Printing text files + # [16-test_cloned_repo_object] + print_file = 'dir1/file2.txt' + tree[print_file] + + # Output + # ![16-test_cloned_repo_object] + + # [17-test_cloned_repo_object] + + # ![17-test_cloned_repo_object] From b0da0a9e53a30dfcefaa7d77fe0bd0104b3a814e Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 9 Jul 2023 21:29:26 -0400 Subject: [PATCH 0494/1790] finished code for quickstart --- doc/source/quickstart.rst | 70 +++++++++++++++++++++++++++++++++++++++ test/test_quick_doc.py | 46 +++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 0a728e485..9d63c5674 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -136,4 +136,74 @@ returns list of :class:`Commit ` objects :end-before: # ![11-test_cloned_repo_object] +Trees & Blobs +************** +Latest Commit Tree +################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [12-test_cloned_repo_object] + :end-before: # ![12-test_cloned_repo_object] + +Any Commit Tree +############### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [13-test_cloned_repo_object] + :end-before: # ![13-test_cloned_repo_object] + +Display level 1 Contents +######################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [14-test_cloned_repo_object] + :end-before: # ![14-test_cloned_repo_object] + +Recurse through the Tree +######################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [15-test_cloned_repo_object] + :end-before: # ![15-test_cloned_repo_object] + +.. code-block:: python + + print_files_from_git(tree) + +.. code-block:: python + + # Output + | Downloads, tree + ----| Downloads/file3.txt, blob + | dir1, tree + ----| dir1/file1.txt, blob + ----| dir1/file2.txt, blob + | file4.txt, blob + + +Print file version +################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [16-test_cloned_repo_object] + :end-before: # ![16-test_cloned_repo_object] + +.. code-block:: python + + blob = tree[print_file] + print(blob.data_stream.read().decode()) + + # Output + # file 2 version 1 + # Update version 2 diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index bb3372905..2a95bfff5 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -120,7 +120,53 @@ def test_cloned_repo_object(self, rw_dir): # file4.txt # ![11-test_cloned_repo_object] + '''Trees and Blobs''' + # Latest commit tree + # [12-test_cloned_repo_object] + tree = repo.tree() + # ![12-test_cloned_repo_object] + + # Previous commit tree + # [13-test_cloned_repo_object] + prev_commits = [c for c in repo.iter_commits('--all', max_count=10)] + tree = prev_commits[0].tree + # ![13-test_cloned_repo_object] + + # Iterating through tree + # [14-test_cloned_repo_object] + tree = repo.tree() + files_dirs = [fd for fd in tree] + files_dirs + + # Output + # [, + # , + # ] + + # ![14-test_cloned_repo_object] + + # [15-test_cloned_repo_object] + def print_files_from_git(tree, delim='-', i=0): + files_dirs = [fd for fd in tree] + for fd in files_dirs: + print(f'{delim if i != 0 else ""}| {fd.path}, {fd.type}') + if fd.type == "tree": + print_files_from_git(fd, delim * 4, i + 1) + + # ![15-test_cloned_repo_object] + + # Printing text files + # [16-test_cloned_repo_object] + print_file = 'dir1/file2.txt' + tree[print_file] + + # Output + # ![16-test_cloned_repo_object] + + # [17-test_cloned_repo_object] + + # ![17-test_cloned_repo_object] From fb35ed1d611113637c52a559d6f77aaadb6d403d Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 9 Jul 2023 21:36:28 -0400 Subject: [PATCH 0495/1790] fixed some indentation --- doc/source/quickstart.rst | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 9d63c5674..1c0832ed5 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -99,41 +99,41 @@ returns list of :class:`Commit ` objects * Untracked files - Lets create a new file + Lets create a new file - .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [7-test_cloned_repo_object] - :end-before: # ![7-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [7-test_cloned_repo_object] + :end-before: # ![7-test_cloned_repo_object] - .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [8-test_cloned_repo_object] - :end-before: # ![8-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [8-test_cloned_repo_object] + :end-before: # ![8-test_cloned_repo_object] * Modified files - .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [9-test_cloned_repo_object] - :end-before: # ![9-test_cloned_repo_object] - - .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [10-test_cloned_repo_object] - :end-before: # ![10-test_cloned_repo_object] - - returns a list of :class:`Diff ` objects - - .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [11-test_cloned_repo_object] - :end-before: # ![11-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [9-test_cloned_repo_object] + :end-before: # ![9-test_cloned_repo_object] + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [10-test_cloned_repo_object] + :end-before: # ![10-test_cloned_repo_object] + + returns a list of :class:`Diff ` objects + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [11-test_cloned_repo_object] + :end-before: # ![11-test_cloned_repo_object] Trees & Blobs From 47c83629cfa0550fae71f2c266bd8b236b63fdc6 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 9 Jul 2023 22:17:03 -0400 Subject: [PATCH 0496/1790] added quickstart to toctree and fixed sphinx warning --- doc/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 69fb573a4..72db8ee5a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,6 +9,7 @@ GitPython Documentation :maxdepth: 2 intro + quickstart tutorial reference roadmap From b7955ed1f1511dd7d873e4198b3372c104102b4f Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 9 Jul 2023 22:17:03 -0400 Subject: [PATCH 0497/1790] added quickstart to toctree to fix sphinx warning --- doc/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index 69fb573a4..72db8ee5a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,6 +9,7 @@ GitPython Documentation :maxdepth: 2 intro + quickstart tutorial reference roadmap From 03d26f0c92055759e296a36f2bde1ff9fb439b29 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 10 Jul 2023 13:51:40 -0400 Subject: [PATCH 0498/1790] Removed code from RST --- doc/source/quickstart.rst | 34 ++++++++++++++-------------------- test/test_quick_doc.py | 25 ++++++++++++++++++++----- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 1c0832ed5..5845bf9e2 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -175,35 +175,29 @@ Recurse through the Tree :start-after: # [15-test_cloned_repo_object] :end-before: # ![15-test_cloned_repo_object] -.. code-block:: python +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [16-test_cloned_repo_object] + :end-before: # ![16-test_cloned_repo_object] - print_files_from_git(tree) -.. code-block:: python - # Output - | Downloads, tree - ----| Downloads/file3.txt, blob - | dir1, tree - ----| dir1/file1.txt, blob - ----| dir1/file2.txt, blob - | file4.txt, blob +Printing text files +#################### -Print file version -################## +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [17-test_cloned_repo_object] + :end-before: # ![17-test_cloned_repo_object] .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 - :start-after: # [16-test_cloned_repo_object] - :end-before: # ![16-test_cloned_repo_object] + :start-after: # [18-test_cloned_repo_object] + :end-before: # ![18-test_cloned_repo_object] -.. code-block:: python - blob = tree[print_file] - print(blob.data_stream.read().decode()) - # Output - # file 2 version 1 - # Update version 2 diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 2a95bfff5..f1d644382 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -156,19 +156,34 @@ def print_files_from_git(tree, delim='-', i=0): # ![15-test_cloned_repo_object] - # Printing text files # [16-test_cloned_repo_object] - print_file = 'dir1/file2.txt' - tree[print_file] + print_files_from_git(tree) - # Output - # ![16-test_cloned_repo_object] + # Output + # | Downloads, tree + # ---- | Downloads / file3.txt, blob + # | dir1, tree + # ---- | dir1 / file1.txt, blob + # ---- | dir1 / file2.txt, blob + # | file4.txt, blob + # # ![16-test_cloned_repo_object] + # Printing text files # [17-test_cloned_repo_object] + print_file = 'dir1/file2.txt' + tree[print_file] + # Output # ![17-test_cloned_repo_object] + # [18-test_cloned_repo_object] + blob = tree[print_file] + print(blob.data_stream.read().decode()) + # Output + # file 2 version 1 + # Update version 2 + # ![18-test_cloned_repo_object] From a0045d8844b937e703156adfbeb496ebc70c8950 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 10 Jul 2023 15:00:06 -0400 Subject: [PATCH 0499/1790] Made variable names more intuitive --- test/test_quick_doc.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index f1d644382..49d9a0f36 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -136,8 +136,8 @@ def test_cloned_repo_object(self, rw_dir): # Iterating through tree # [14-test_cloned_repo_object] tree = repo.tree() - files_dirs = [fd for fd in tree] - files_dirs + files_and_dirs = [entry for entry in tree] + files_and_dirs # Output # [, @@ -147,12 +147,11 @@ def test_cloned_repo_object(self, rw_dir): # ![14-test_cloned_repo_object] # [15-test_cloned_repo_object] - def print_files_from_git(tree, delim='-', i=0): - files_dirs = [fd for fd in tree] - for fd in files_dirs: - print(f'{delim if i != 0 else ""}| {fd.path}, {fd.type}') - if fd.type == "tree": - print_files_from_git(fd, delim * 4, i + 1) + def print_files_from_git(root, delim='-', i=0): + for entry in root: + print(f'{delim if i != 0 else ""}| {entry.path}, {entry.type}') + if entry.type == "tree": + print_files_from_git(entry, delim * 4, i + 1) # ![15-test_cloned_repo_object] From 98336551260f0b3093f5be085573b193198a4271 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 10 Jul 2023 15:19:00 -0400 Subject: [PATCH 0500/1790] Updated the sample repo URL --- test/test_quick_doc.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 49d9a0f36..e76bf3a12 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -12,8 +12,7 @@ def tearDown(self): gc.collect() @with_rw_directory - def test_init_repo_object(self, rw_dir): - path_to_dir = rw_dir + def test_init_repo_object(self, path_to_dir): # [1-test_init_repo_object] from git import Repo @@ -32,19 +31,15 @@ def test_init_repo_object(self, rw_dir): # ![2-test_init_repo_object] @with_rw_directory - def test_cloned_repo_object(self, rw_dir): - local_dir = rw_dir + def test_cloned_repo_object(self, local_dir): from git import Repo import git # code to clone from url # [1-test_cloned_repo_object] - repo_url = "https://github.com/LeoDaCoda/GitPython-TestFileSys.git" + repo_url = "https://github.com/gitpython-developers/QuickStartTutorialFiles.git" - try: - repo = Repo.clone_from(repo_url, local_dir) - except git.CommandError: - assert False, f"Invalid address {repo_url}" + repo = Repo.clone_from(repo_url, local_dir) # ![1-test_cloned_repo_object] # code to add files From 3cda530b1fc1e5ae3c2403a43a7270f6a73f07fb Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 10 Jul 2023 15:24:05 -0400 Subject: [PATCH 0501/1790] removed try/except and updated sample url --- doc/source/quickstart.rst | 2 +- test/test_quick_doc.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 5845bf9e2..5c1c18701 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -38,7 +38,7 @@ Clone from URL For the rest of this tutorial we will use a clone from https://github.com/LeoDaCoda/GitPython-TestFileSys.git -git clone https://some_repo_url +$ git clone https://github.com/gitpython-developers/QuickStartTutorialFiles.git .. literalinclude:: ../../test/test_quick_doc.py :language: python diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index e76bf3a12..64586f186 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -18,16 +18,10 @@ def test_init_repo_object(self, path_to_dir): from git import Repo repo = Repo.init(path_to_dir) # git init path/to/dir - assert repo.__class__ is Repo # Test to confirm repo was initialized - # ![1-test_init_repo_object] + # ![1-test_init_repo_object] # [2-test_init_repo_object] - import git - - try: - repo = Repo(path_to_dir) - except git.NoSuchPathError: - assert False, f"No such path {path_to_dir}" + repo = Repo(path_to_dir) # ![2-test_init_repo_object] @with_rw_directory From e4bbc7a520d83b7e5db208d0fe901cec0125c2f9 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 10 Jul 2023 17:13:44 -0400 Subject: [PATCH 0502/1790] correct way to get the latest commit tree --- test/test_quick_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 64586f186..f8c973bad 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -113,7 +113,7 @@ def test_cloned_repo_object(self, local_dir): # Latest commit tree # [12-test_cloned_repo_object] - tree = repo.tree() + tree = repo.head.commit.tree # ![12-test_cloned_repo_object] # Previous commit tree From a1dfd4ade535242bb535cbda9b2f02153d2a423e Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 10 Jul 2023 23:56:06 -0400 Subject: [PATCH 0503/1790] convert from --all flag to all=True --- test/test_quick_doc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index f8c973bad..701d3994a 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -57,14 +57,14 @@ def test_cloned_repo_object(self, local_dir): # [5-test_cloned_repo_object] file = 'dir1/file2.txt' # relative path from git root - repo.iter_commits('--all', max_count=100, paths=file) + repo.iter_commits(all=True, max_count=10, paths=file) # gets the last 10 commits from all branches # Outputs: # ![5-test_cloned_repo_object] # [6-test_cloned_repo_object] - commits_for_file_generator = repo.iter_commits('--all', max_count=100, paths=file) + commits_for_file_generator = repo.iter_commits(all=True, max_count=10, paths=file) commits_for_file = [c for c in commits_for_file_generator] commits_for_file @@ -95,7 +95,8 @@ def test_cloned_repo_object(self, local_dir): # ![9-test_cloned_repo_object] # [10-test_cloned_repo_object] - repo.index.diff(None) + repo.index.diff(None) # compares staging area to working directory + repo.index.diff(repo.head.commit) # compares staging area to last commit # Output: [, # ] # ![10-test_cloned_repo_object] @@ -118,7 +119,7 @@ def test_cloned_repo_object(self, local_dir): # Previous commit tree # [13-test_cloned_repo_object] - prev_commits = [c for c in repo.iter_commits('--all', max_count=10)] + prev_commits = [c for c in repo.iter_commits(all=True, max_count=10)] # last 10 commits from all branches tree = prev_commits[0].tree # ![13-test_cloned_repo_object] From a8b58639f57f5a4952f98ee097def5ad9543b566 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 11 Jul 2023 00:02:20 -0400 Subject: [PATCH 0504/1790] removed unnecessary variables --- test/test_quick_doc.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 701d3994a..9756f0da5 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -46,7 +46,7 @@ def test_cloned_repo_object(self, local_dir): # ![2-test_cloned_repo_object] # [3-test_cloned_repo_object] - add_file = [f"{update_file}"] # relative path from git root + add_file = [update_file] # relative path from git root repo.index.add(add_file) # notice the add function requires a list of paths # ![3-test_cloned_repo_object] @@ -56,15 +56,15 @@ def test_cloned_repo_object(self, local_dir): # ![4-test_cloned_repo_object] # [5-test_cloned_repo_object] - file = 'dir1/file2.txt' # relative path from git root - repo.iter_commits(all=True, max_count=10, paths=file) # gets the last 10 commits from all branches + # relative path from git root + repo.iter_commits(all=True, max_count=10, paths=update_file) # gets the last 10 commits from all branches # Outputs: # ![5-test_cloned_repo_object] # [6-test_cloned_repo_object] - commits_for_file_generator = repo.iter_commits(all=True, max_count=10, paths=file) + commits_for_file_generator = repo.iter_commits(all=True, max_count=10, paths=update_file) commits_for_file = [c for c in commits_for_file_generator] commits_for_file @@ -76,8 +76,7 @@ def test_cloned_repo_object(self, local_dir): # [7-test_cloned_repo_object] # We'll create a file5.txt - file5 = f'{local_dir}/file5.txt' - with open(file5, 'w') as f: + with open(f'{local_dir}/file5.txt', 'w') as f: f.write('file5 version 1') # ![7-test_cloned_repo_object] @@ -89,8 +88,8 @@ def test_cloned_repo_object(self, local_dir): # Modified files # [9-test_cloned_repo_object] # Lets modify one of our tracked files - file3 = f'{local_dir}/Downloads/file3.txt' - with open(file3, 'w') as f: + + with open(f'{local_dir}/Downloads/file3.txt', 'w') as f: f.write('file3 version 2') # overwrite file 3 # ![9-test_cloned_repo_object] From abe7e6e5075ba4b9ea4cfc74b6121ad977dc7e4f Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 11 Jul 2023 00:23:18 -0400 Subject: [PATCH 0505/1790] replaced output cell to generic commit ID --- test/test_quick_doc.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 9756f0da5..0f09d4ed9 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -68,8 +68,8 @@ def test_cloned_repo_object(self, local_dir): commits_for_file = [c for c in commits_for_file_generator] commits_for_file - # Outputs: [, - # ] + # Outputs: [, + # ] # ![6-test_cloned_repo_object] # Untracked files - create new file @@ -124,14 +124,13 @@ def test_cloned_repo_object(self, local_dir): # Iterating through tree # [14-test_cloned_repo_object] - tree = repo.tree() files_and_dirs = [entry for entry in tree] files_and_dirs # Output - # [, - # , - # ] + # [, + # , + # ] # ![14-test_cloned_repo_object] From 1369bdc6d7d06e473b7c211a4070dcee94438e64 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Thu, 13 Jul 2023 01:35:41 -0400 Subject: [PATCH 0506/1790] replaced hash with generic --- test/test_quick_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 0f09d4ed9..d1aea44e7 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -160,7 +160,7 @@ def print_files_from_git(root, delim='-', i=0): print_file = 'dir1/file2.txt' tree[print_file] - # Output + # Output # ![17-test_cloned_repo_object] # [18-test_cloned_repo_object] From 9cd9431906acff137e441a2dd82d1d6d4e6322d7 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Thu, 13 Jul 2023 01:36:17 -0400 Subject: [PATCH 0507/1790] draft of description --- doc/source/quickstart.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 5c1c18701..018e13a76 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -7,6 +7,9 @@ ============================== GitPython Quick Start Tutorial ============================== +Welcome to the GitPython Quickstart Guide! Designed for developers seeking a practical and interactive learning experience, this concise resource offers step-by-step code snippets to swiftly initialize/clone repositories, perform essential Git operations, and explore GitPython's capabilities. Get ready to dive in, experiment, and unleash the power of GitPython in your projects! + +All code presented here originated from `***** insert link **** `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes. All you need is a developer installation of git-python. git.Repo ******** From 393bae5184abda11cdaab128049fccba2fcb213f Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Thu, 13 Jul 2023 01:42:24 -0400 Subject: [PATCH 0508/1790] clarified comment --- doc/source/quickstart.rst | 2 +- test/test_quick_doc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 018e13a76..bd5ddd7b8 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -39,7 +39,7 @@ Existing local git Repo Clone from URL ############## -For the rest of this tutorial we will use a clone from https://github.com/LeoDaCoda/GitPython-TestFileSys.git +For the rest of this tutorial we will use a clone from https://github.com/gitpython-developers/QuickStartTutorialFiles.git $ git clone https://github.com/gitpython-developers/QuickStartTutorialFiles.git diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index d1aea44e7..3dc29a22c 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -40,7 +40,7 @@ def test_cloned_repo_object(self, local_dir): # [2-test_cloned_repo_object] # We must make a change to a file so that we can add the update to git - update_file = 'dir1/file2.txt' # we'll use ./dir1/file2.txt + update_file = 'dir1/file2.txt' # we'll use local_dir/dir1/file2.txt with open(f"{local_dir}/{update_file}", 'a') as f: f.write('\nUpdate version 2') # ![2-test_cloned_repo_object] From aa6d27f9204d68b21cd24366c8a58fb4f9578553 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Thu, 13 Jul 2023 16:15:16 -0400 Subject: [PATCH 0509/1790] refactored print git tree --- test/test_quick_doc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 3dc29a22c..0aa27a361 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -135,11 +135,11 @@ def test_cloned_repo_object(self, local_dir): # ![14-test_cloned_repo_object] # [15-test_cloned_repo_object] - def print_files_from_git(root, delim='-', i=0): + def print_files_from_git(root, level=0): for entry in root: - print(f'{delim if i != 0 else ""}| {entry.path}, {entry.type}') + print(f'{"-" * 4 * level}| {entry.path}, {entry.type}') if entry.type == "tree": - print_files_from_git(entry, delim * 4, i + 1) + print_files_from_git(entry, level + 1) # ![15-test_cloned_repo_object] @@ -148,10 +148,10 @@ def print_files_from_git(root, delim='-', i=0): # Output # | Downloads, tree - # ---- | Downloads / file3.txt, blob + # ----| Downloads / file3.txt, blob # | dir1, tree - # ---- | dir1 / file1.txt, blob - # ---- | dir1 / file2.txt, blob + # ----| dir1 / file1.txt, blob + # ----| dir1 / file2.txt, blob # | file4.txt, blob # # ![16-test_cloned_repo_object] From 6d78ff1ac33fa2adeb0518feb33a634c09b0b5b5 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 16 Jul 2023 11:58:46 -0400 Subject: [PATCH 0510/1790] Made trees and blobs the first section --- doc/source/quickstart.rst | 133 +++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index bd5ddd7b8..11f8123bb 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -11,6 +11,74 @@ Welcome to the GitPython Quickstart Guide! Designed for developers seeking a pra All code presented here originated from `***** insert link **** `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes. All you need is a developer installation of git-python. + +Trees & Blobs +************** + +Latest Commit Tree +################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [12-test_cloned_repo_object] + :end-before: # ![12-test_cloned_repo_object] + +Any Commit Tree +############### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [13-test_cloned_repo_object] + :end-before: # ![13-test_cloned_repo_object] + +Display level 1 Contents +######################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [14-test_cloned_repo_object] + :end-before: # ![14-test_cloned_repo_object] + +Recurse through the Tree +######################## + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [15-test_cloned_repo_object] + :end-before: # ![15-test_cloned_repo_object] + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [16-test_cloned_repo_object] + :end-before: # ![16-test_cloned_repo_object] + + + + +Printing text files +#################### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [17-test_cloned_repo_object] + :end-before: # ![17-test_cloned_repo_object] + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [18-test_cloned_repo_object] + :end-before: # ![18-test_cloned_repo_object] + + + + + git.Repo ******** @@ -139,68 +207,3 @@ returns list of :class:`Commit ` objects :end-before: # ![11-test_cloned_repo_object] -Trees & Blobs -************** - -Latest Commit Tree -################## - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [12-test_cloned_repo_object] - :end-before: # ![12-test_cloned_repo_object] - -Any Commit Tree -############### - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [13-test_cloned_repo_object] - :end-before: # ![13-test_cloned_repo_object] - -Display level 1 Contents -######################## - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [14-test_cloned_repo_object] - :end-before: # ![14-test_cloned_repo_object] - -Recurse through the Tree -######################## - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [15-test_cloned_repo_object] - :end-before: # ![15-test_cloned_repo_object] - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [16-test_cloned_repo_object] - :end-before: # ![16-test_cloned_repo_object] - - - - -Printing text files -#################### - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [17-test_cloned_repo_object] - :end-before: # ![17-test_cloned_repo_object] - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [18-test_cloned_repo_object] - :end-before: # ![18-test_cloned_repo_object] - - - From 2c9c0c122d7dddce62d593f564d2b0c6f7a33e69 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 16 Jul 2023 12:05:18 -0400 Subject: [PATCH 0511/1790] Added warning about index add --- doc/source/quickstart.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 11f8123bb..693562b17 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -138,6 +138,8 @@ Now lets add the updated file to git Notice the add method requires a list as a parameter +Warning: If you experience any trouble with this, try to invoke :class:`git ` instead via repo.git.add(path) + * $ git commit -m message .. literalinclude:: ../../test/test_quick_doc.py From d276107039d69bb3ad32595756b70fd4e51267d1 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 16 Jul 2023 12:08:19 -0400 Subject: [PATCH 0512/1790] Updated generic sha hash --- test/test_quick_doc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 0aa27a361..5aa5664bc 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -68,8 +68,8 @@ def test_cloned_repo_object(self, local_dir): commits_for_file = [c for c in commits_for_file_generator] commits_for_file - # Outputs: [, - # ] + # Outputs: [, + # ] # ![6-test_cloned_repo_object] # Untracked files - create new file @@ -128,9 +128,9 @@ def test_cloned_repo_object(self, local_dir): files_and_dirs # Output - # [, - # , - # ] + # [, + # , + # ] # ![14-test_cloned_repo_object] From f3968f2e34467735935ee7a39a7d2b2f07229e7d Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 16 Jul 2023 12:19:44 -0400 Subject: [PATCH 0513/1790] Removed all reference to source code --- doc/source/quickstart.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 693562b17..0cbb4f45c 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -9,8 +9,6 @@ GitPython Quick Start Tutorial ============================== Welcome to the GitPython Quickstart Guide! Designed for developers seeking a practical and interactive learning experience, this concise resource offers step-by-step code snippets to swiftly initialize/clone repositories, perform essential Git operations, and explore GitPython's capabilities. Get ready to dive in, experiment, and unleash the power of GitPython in your projects! -All code presented here originated from `***** insert link **** `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes. All you need is a developer installation of git-python. - Trees & Blobs ************** From 9ca25d767e554681ad9863138911800868c29b49 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 16 Jul 2023 13:30:09 -0400 Subject: [PATCH 0514/1790] WIP major changes to structure to improve readability --- doc/source/quickstart.rst | 122 ++++++++++++++++++++------------------ test/test_quick_doc.py | 27 ++++++++- 2 files changed, 87 insertions(+), 62 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 0cbb4f45c..f33d51600 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -10,6 +10,41 @@ GitPython Quick Start Tutorial Welcome to the GitPython Quickstart Guide! Designed for developers seeking a practical and interactive learning experience, this concise resource offers step-by-step code snippets to swiftly initialize/clone repositories, perform essential Git operations, and explore GitPython's capabilities. Get ready to dive in, experiment, and unleash the power of GitPython in your projects! +git.Repo +******** + +There are a few ways to create a :class:`git.Repo ` object + +Initialize a new git Repo +######################### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [1-test_init_repo_object] + :end-before: # ![1-test_init_repo_object] + +Existing local git Repo +####################### + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [2-test_init_repo_object] + :end-before: # ![2-test_init_repo_object] + +Clone from URL +############## + +For the rest of this tutorial we will use a clone from https://github.com/gitpython-developers/QuickStartTutorialFiles.git + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [1-test_cloned_repo_object] + :end-before: # ![1-test_cloned_repo_object] + + Trees & Blobs ************** @@ -40,6 +75,12 @@ Display level 1 Contents :start-after: # [14-test_cloned_repo_object] :end-before: # ![14-test_cloned_repo_object] +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [14.1-test_cloned_repo_object] + :end-before: # ![14.1-test_cloned_repo_object] + Recurse through the Tree ######################## @@ -58,67 +99,10 @@ Recurse through the Tree -Printing text files -#################### - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [17-test_cloned_repo_object] - :end-before: # ![17-test_cloned_repo_object] - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [18-test_cloned_repo_object] - :end-before: # ![18-test_cloned_repo_object] - - - - - -git.Repo -******** - -There are a few ways to create a :class:`git.Repo ` object - -An existing local path -###################### - -$ git init path/to/dir - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [1-test_init_repo_object] - :end-before: # ![1-test_init_repo_object] - -Existing local git Repo -####################### - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [2-test_init_repo_object] - :end-before: # ![2-test_init_repo_object] - -Clone from URL -############## - -For the rest of this tutorial we will use a clone from https://github.com/gitpython-developers/QuickStartTutorialFiles.git - -$ git clone https://github.com/gitpython-developers/QuickStartTutorialFiles.git - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [1-test_cloned_repo_object] - :end-before: # ![1-test_cloned_repo_object] - Usage **************** -* $ git add filepath +* $ git add .. literalinclude:: ../../test/test_quick_doc.py :language: python @@ -146,7 +130,7 @@ Warning: If you experience any trouble with this, try to invoke :class:`git A list of commits associated with a file @@ -166,6 +150,24 @@ Notice this returns a generator object returns list of :class:`Commit ` objects +Printing text files +#################### +Lets print the latest version of ` dir1/file2.txt` + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [17-test_cloned_repo_object] + :end-before: # ![17-test_cloned_repo_object] + +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [18-test_cloned_repo_object] + :end-before: # ![18-test_cloned_repo_object] + +Previous version of `/dir1/file2.txt` + * $ git status * Untracked files @@ -207,3 +209,5 @@ returns list of :class:`Commit ` objects :end-before: # ![11-test_cloned_repo_object] + + diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 5aa5664bc..61b8082d0 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -15,6 +15,8 @@ def tearDown(self): def test_init_repo_object(self, path_to_dir): # [1-test_init_repo_object] + # $ git init + from git import Repo repo = Repo.init(path_to_dir) # git init path/to/dir @@ -31,6 +33,8 @@ def test_cloned_repo_object(self, local_dir): import git # code to clone from url # [1-test_cloned_repo_object] + # $ git clone + repo_url = "https://github.com/gitpython-developers/QuickStartTutorialFiles.git" repo = Repo.clone_from(repo_url, local_dir) @@ -128,12 +132,22 @@ def test_cloned_repo_object(self, local_dir): files_and_dirs # Output - # [, - # , - # ] + # [, + # , + # ] # ![14-test_cloned_repo_object] + # [14.1-test_cloned_repo_object] + files_and_dirs = [(entry, entry.name) for entry in tree] + files_and_dirs + + # Output + # [(< git.Tree "SHA1-HEX_HASH" >, 'Downloads', 'tree'), + # (< git.Tree "SHA1-HEX_HASH" >, 'dir1', 'tree'), + # (< git.Blob "SHA1-HEX_HASH" >, 'file4.txt', 'blob')] + # ![14.1-test_cloned_repo_object] + # [15-test_cloned_repo_object] def print_files_from_git(root, level=0): for entry in root: @@ -163,6 +177,13 @@ def print_files_from_git(root, level=0): # Output # ![17-test_cloned_repo_object] + # print pre + # [17.1-test_cloned_repo_object] + commits_for_file = [c for c in repo.iter_commits(all=True, paths=print_file)] + blob = tree[print_file] + + # ![17.1-test_cloned_repo_object] + # [18-test_cloned_repo_object] blob = tree[print_file] print(blob.data_stream.read().decode()) From 7fa57e5e30e56e7aa247cf77d1b84ddf5d08d1e7 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Sun, 16 Jul 2023 18:33:24 -0400 Subject: [PATCH 0515/1790] Added new section to print prev file --- doc/source/quickstart.rst | 6 ++++++ test/test_quick_doc.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index f33d51600..29c500a72 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -168,6 +168,12 @@ Lets print the latest version of ` dir1/file2.txt` Previous version of `/dir1/file2.txt` +.. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [18.1-test_cloned_repo_object] + :end-before: # ![18.1-test_cloned_repo_object] + * $ git status * Untracked files diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 61b8082d0..0397eb6d9 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -177,13 +177,7 @@ def print_files_from_git(root, level=0): # Output # ![17-test_cloned_repo_object] - # print pre - # [17.1-test_cloned_repo_object] - commits_for_file = [c for c in repo.iter_commits(all=True, paths=print_file)] - blob = tree[print_file] - - # ![17.1-test_cloned_repo_object] - + # print latest file # [18-test_cloned_repo_object] blob = tree[print_file] print(blob.data_stream.read().decode()) @@ -191,7 +185,16 @@ def print_files_from_git(root, level=0): # Output # file 2 version 1 # Update version 2 - # ![18-test_cloned_repo_object] + # print previous tree + # [18.1-test_cloned_repo_object] + commits_for_file = [c for c in repo.iter_commits(all=True, paths=print_file)] + tree = commits_for_file[-1].tree # gets the first commit tree + blob = tree[print_file] + + print(blob.data_stream.read().decode()) + # Output + # file 2 version 1 + # ![18.1-test_cloned_repo_object] \ No newline at end of file From 9d878af964947a09e74f29e3a13b5a26d606e86f Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 17 Jul 2023 14:21:17 -0400 Subject: [PATCH 0516/1790] change to formatting - removed = bash cmds --- doc/source/quickstart.rst | 166 +++++++++++++++++++------------------- test/test_quick_doc.py | 12 +-- 2 files changed, 91 insertions(+), 87 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 29c500a72..01a664e9c 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -102,117 +102,119 @@ Recurse through the Tree Usage **************** -* $ git add - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [2-test_cloned_repo_object] - :end-before: # ![2-test_cloned_repo_object] - -Now lets add the updated file to git - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [3-test_cloned_repo_object] - :end-before: # ![3-test_cloned_repo_object] - -Notice the add method requires a list as a parameter - -Warning: If you experience any trouble with this, try to invoke :class:`git ` instead via repo.git.add(path) - -* $ git commit -m message - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [4-test_cloned_repo_object] - :end-before: # ![4-test_cloned_repo_object] - -* $ git log - -A list of commits associated with a file - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [5-test_cloned_repo_object] - :end-before: # ![5-test_cloned_repo_object] - -Notice this returns a generator object - -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [6-test_cloned_repo_object] - :end-before: # ![6-test_cloned_repo_object] - -returns list of :class:`Commit ` objects - -Printing text files -#################### -Lets print the latest version of ` dir1/file2.txt` +Add file to staging area +######################## -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [17-test_cloned_repo_object] - :end-before: # ![17-test_cloned_repo_object] -.. literalinclude:: ../../test/test_quick_doc.py + .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 - :start-after: # [18-test_cloned_repo_object] - :end-before: # ![18-test_cloned_repo_object] + :start-after: # [2-test_cloned_repo_object] + :end-before: # ![2-test_cloned_repo_object] -Previous version of `/dir1/file2.txt` + Now lets add the updated file to git -.. literalinclude:: ../../test/test_quick_doc.py + .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 - :start-after: # [18.1-test_cloned_repo_object] - :end-before: # ![18.1-test_cloned_repo_object] + :start-after: # [3-test_cloned_repo_object] + :end-before: # ![3-test_cloned_repo_object] -* $ git status + Notice the add method requires a list as a parameter - * Untracked files + Warning: If you experience any trouble with this, try to invoke :class:`git ` instead via repo.git.add(path) - Lets create a new file +Commit +###### .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 - :start-after: # [7-test_cloned_repo_object] - :end-before: # ![7-test_cloned_repo_object] + :start-after: # [4-test_cloned_repo_object] + :end-before: # ![4-test_cloned_repo_object] + +List of commits associated with a file +####################################### .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 - :start-after: # [8-test_cloned_repo_object] - :end-before: # ![8-test_cloned_repo_object] + :start-after: # [5-test_cloned_repo_object] + :end-before: # ![5-test_cloned_repo_object] - * Modified files + Notice this returns a generator object .. literalinclude:: ../../test/test_quick_doc.py :language: python :dedent: 8 - :start-after: # [9-test_cloned_repo_object] - :end-before: # ![9-test_cloned_repo_object] + :start-after: # [6-test_cloned_repo_object] + :end-before: # ![6-test_cloned_repo_object] + + returns list of :class:`Commit ` objects + +Printing text files +#################### +Lets print the latest version of ` dir1/file2.txt` .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [10-test_cloned_repo_object] - :end-before: # ![10-test_cloned_repo_object] + :language: python + :dedent: 8 + :start-after: # [17-test_cloned_repo_object] + :end-before: # ![17-test_cloned_repo_object] + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [18-test_cloned_repo_object] + :end-before: # ![18-test_cloned_repo_object] - returns a list of :class:`Diff ` objects + Previous version of `/dir1/file2.txt` .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [11-test_cloned_repo_object] - :end-before: # ![11-test_cloned_repo_object] + :language: python + :dedent: 8 + :start-after: # [18.1-test_cloned_repo_object] + :end-before: # ![18.1-test_cloned_repo_object] + +Status +###### + * Untracked files + + Lets create a new file + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [7-test_cloned_repo_object] + :end-before: # ![7-test_cloned_repo_object] + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [8-test_cloned_repo_object] + :end-before: # ![8-test_cloned_repo_object] + + * Modified files + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [9-test_cloned_repo_object] + :end-before: # ![9-test_cloned_repo_object] + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [10-test_cloned_repo_object] + :end-before: # ![10-test_cloned_repo_object] + + returns a list of :class:`Diff ` objects + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [11-test_cloned_repo_object] + :end-before: # ![11-test_cloned_repo_object] diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 0397eb6d9..4ab2a59a6 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -50,16 +50,20 @@ def test_cloned_repo_object(self, local_dir): # ![2-test_cloned_repo_object] # [3-test_cloned_repo_object] + # $ git add add_file = [update_file] # relative path from git root repo.index.add(add_file) # notice the add function requires a list of paths # ![3-test_cloned_repo_object] # code to commit - not sure how to test this # [4-test_cloned_repo_object] + # $ git commit -m repo.index.commit("Update to file2") # ![4-test_cloned_repo_object] # [5-test_cloned_repo_object] + # $ git log + # relative path from git root repo.iter_commits(all=True, max_count=10, paths=update_file) # gets the last 10 commits from all branches @@ -78,15 +82,13 @@ def test_cloned_repo_object(self, local_dir): # Untracked files - create new file # [7-test_cloned_repo_object] - # We'll create a file5.txt - - with open(f'{local_dir}/file5.txt', 'w') as f: - f.write('file5 version 1') + f = open(f'{local_dir}/untracked.txt', 'w') # creates an empty file + f.close() # ![7-test_cloned_repo_object] # [8-test_cloned_repo_object] repo.untracked_files - # Output: ['file5.txt'] + # Output: ['untracked.txt'] # ![8-test_cloned_repo_object] # Modified files From 315405d9395ff94348d43912d15471e6dd465100 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Mon, 17 Jul 2023 18:49:34 -0400 Subject: [PATCH 0517/1790] formatting wip --- test/test_quick_doc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 4ab2a59a6..c09845a6a 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -93,7 +93,7 @@ def test_cloned_repo_object(self, local_dir): # Modified files # [9-test_cloned_repo_object] - # Lets modify one of our tracked files + # Let's modify one of our tracked files with open(f'{local_dir}/Downloads/file3.txt', 'w') as f: f.write('file3 version 2') # overwrite file 3 @@ -174,7 +174,7 @@ def print_files_from_git(root, level=0): # Printing text files # [17-test_cloned_repo_object] print_file = 'dir1/file2.txt' - tree[print_file] + tree[print_file] # the head commit tree # Output # ![17-test_cloned_repo_object] From bccf8bc3ee2384048548e717e64a5d42156ba236 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 18 Jul 2023 00:16:07 -0400 Subject: [PATCH 0518/1790] added new section for diffs and formatting --- doc/source/quickstart.rst | 24 ++++++++++++++++++++++++ test/test_quick_doc.py | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 01a664e9c..0826dec29 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -216,6 +216,30 @@ Status :start-after: # [11-test_cloned_repo_object] :end-before: # ![11-test_cloned_repo_object] +Diffs +###### + + Compare staging area to head commit + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [11.1-test_cloned_repo_object] + :end-before: # ![11.1-test_cloned_repo_object] + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [11.2-test_cloned_repo_object] + :end-before: # ![11.2-test_cloned_repo_object] + + Compare commit to commit + + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [11.3-test_cloned_repo_object] + :end-before: # ![11.3-test_cloned_repo_object] diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index c09845a6a..f79a9645e 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -19,7 +19,7 @@ def test_init_repo_object(self, path_to_dir): from git import Repo - repo = Repo.init(path_to_dir) # git init path/to/dir + repo = Repo.init(path_to_dir) # ![1-test_init_repo_object] # [2-test_init_repo_object] @@ -111,10 +111,43 @@ def test_cloned_repo_object(self, local_dir): for d in diffs: print(d.a_path) + # Output # Downloads/file3.txt - # file4.txt # ![11-test_cloned_repo_object] + # compares staging area to head commit + # [11.1-test_cloned_repo_object] + diffs = repo.index.diff(repo.head.commit) + for d in diffs: + print(d.a_path) + + # Output + + # ![11.1-test_cloned_repo_object] + # [11.2-test_cloned_repo_object] + # lets add untracked.txt + repo.index.add(['untracked.txt']) + diffs = repo.index.diff(repo.head.commit) + for d in diffs: + print(d.a_path) + + # Output + # untracked.txt + # ![11.2-test_cloned_repo_object] + + # Compare commit to commit + # [11.3-test_cloned_repo_object] + first_commit = [c for c in repo.iter_commits(all=True)][-1] + diffs = repo.head.commit.diff(first_commit) + for d in diffs: + print(d.a_path) + + # Output + # dir1/file2.txt + # ![11.3-test_cloned_repo_object] + + + '''Trees and Blobs''' # Latest commit tree @@ -141,7 +174,7 @@ def test_cloned_repo_object(self, local_dir): # ![14-test_cloned_repo_object] # [14.1-test_cloned_repo_object] - files_and_dirs = [(entry, entry.name) for entry in tree] + files_and_dirs = [(entry, entry.name, entry.type) for entry in tree] files_and_dirs # Output From cad1e2e835b0b7876277c0514bcba2ac6fedab81 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 18 Jul 2023 00:19:21 -0400 Subject: [PATCH 0519/1790] tabbed all code-blocks --- doc/source/quickstart.rst | 90 +++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 0826dec29..2b6c1c99f 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -18,31 +18,31 @@ There are a few ways to create a :class:`git.Repo ` object Initialize a new git Repo ######################### -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [1-test_init_repo_object] - :end-before: # ![1-test_init_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [1-test_init_repo_object] + :end-before: # ![1-test_init_repo_object] Existing local git Repo ####################### -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [2-test_init_repo_object] - :end-before: # ![2-test_init_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [2-test_init_repo_object] + :end-before: # ![2-test_init_repo_object] Clone from URL ############## For the rest of this tutorial we will use a clone from https://github.com/gitpython-developers/QuickStartTutorialFiles.git -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [1-test_cloned_repo_object] - :end-before: # ![1-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [1-test_cloned_repo_object] + :end-before: # ![1-test_cloned_repo_object] Trees & Blobs @@ -51,50 +51,50 @@ Trees & Blobs Latest Commit Tree ################## -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [12-test_cloned_repo_object] - :end-before: # ![12-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [12-test_cloned_repo_object] + :end-before: # ![12-test_cloned_repo_object] Any Commit Tree ############### -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [13-test_cloned_repo_object] - :end-before: # ![13-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [13-test_cloned_repo_object] + :end-before: # ![13-test_cloned_repo_object] Display level 1 Contents ######################## -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [14-test_cloned_repo_object] - :end-before: # ![14-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [14-test_cloned_repo_object] + :end-before: # ![14-test_cloned_repo_object] -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [14.1-test_cloned_repo_object] - :end-before: # ![14.1-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [14.1-test_cloned_repo_object] + :end-before: # ![14.1-test_cloned_repo_object] Recurse through the Tree ######################## -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [15-test_cloned_repo_object] - :end-before: # ![15-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [15-test_cloned_repo_object] + :end-before: # ![15-test_cloned_repo_object] -.. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [16-test_cloned_repo_object] - :end-before: # ![16-test_cloned_repo_object] + .. literalinclude:: ../../test/test_quick_doc.py + :language: python + :dedent: 8 + :start-after: # [16-test_cloned_repo_object] + :end-before: # ![16-test_cloned_repo_object] From 7e589f3d852461e2c143035c1cc3ceb1a81ecd61 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 18 Jul 2023 00:29:44 -0400 Subject: [PATCH 0520/1790] fixed tabbing --- doc/source/quickstart.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 2b6c1c99f..ebebc37d1 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -219,7 +219,7 @@ Status Diffs ###### - Compare staging area to head commit +Compare staging area to head commit .. literalinclude:: ../../test/test_quick_doc.py :language: python @@ -233,7 +233,7 @@ Diffs :start-after: # [11.2-test_cloned_repo_object] :end-before: # ![11.2-test_cloned_repo_object] - Compare commit to commit +Compare commit to commit .. literalinclude:: ../../test/test_quick_doc.py :language: python From 2a45f94d976e3cb91a7e700649eeea12f6655f7c Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 18 Jul 2023 00:38:38 -0400 Subject: [PATCH 0521/1790] redundant line --- test/test_quick_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index f79a9645e..cea96690e 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -101,7 +101,7 @@ def test_cloned_repo_object(self, local_dir): # [10-test_cloned_repo_object] repo.index.diff(None) # compares staging area to working directory - repo.index.diff(repo.head.commit) # compares staging area to last commit + # Output: [, # ] # ![10-test_cloned_repo_object] From ef4d6d52fe02b7006224765cb65c824b8eca91e5 Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 18 Jul 2023 15:18:24 -0400 Subject: [PATCH 0522/1790] redundant code cell --- doc/source/quickstart.rst | 7 ------- test/test_quick_doc.py | 13 +------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index ebebc37d1..33ddf5901 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -75,12 +75,6 @@ Display level 1 Contents :start-after: # [14-test_cloned_repo_object] :end-before: # ![14-test_cloned_repo_object] - .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [14.1-test_cloned_repo_object] - :end-before: # ![14.1-test_cloned_repo_object] - Recurse through the Tree ######################## @@ -242,4 +236,3 @@ Compare commit to commit :end-before: # ![11.3-test_cloned_repo_object] - diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index cea96690e..cb782aa3c 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -163,17 +163,6 @@ def test_cloned_repo_object(self, local_dir): # Iterating through tree # [14-test_cloned_repo_object] - files_and_dirs = [entry for entry in tree] - files_and_dirs - - # Output - # [, - # , - # ] - - # ![14-test_cloned_repo_object] - - # [14.1-test_cloned_repo_object] files_and_dirs = [(entry, entry.name, entry.type) for entry in tree] files_and_dirs @@ -181,7 +170,7 @@ def test_cloned_repo_object(self, local_dir): # [(< git.Tree "SHA1-HEX_HASH" >, 'Downloads', 'tree'), # (< git.Tree "SHA1-HEX_HASH" >, 'dir1', 'tree'), # (< git.Blob "SHA1-HEX_HASH" >, 'file4.txt', 'blob')] - # ![14.1-test_cloned_repo_object] + # ![14-test_cloned_repo_object] # [15-test_cloned_repo_object] def print_files_from_git(root, level=0): From 8138b3a56d16f68cfe6a5d9371e2fde3d587161c Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Tue, 18 Jul 2023 15:25:43 -0400 Subject: [PATCH 0523/1790] generic hash --- test/test_quick_doc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index cb782aa3c..eaee4e581 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -77,7 +77,7 @@ def test_cloned_repo_object(self, local_dir): commits_for_file # Outputs: [, - # ] + # ] # ![6-test_cloned_repo_object] # Untracked files - create new file @@ -198,7 +198,7 @@ def print_files_from_git(root, level=0): print_file = 'dir1/file2.txt' tree[print_file] # the head commit tree - # Output + # Output # ![17-test_cloned_repo_object] # print latest file From 84885a3ea412261adf457aee1c6471606ba7095c Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Wed, 19 Jul 2023 13:47:28 -0400 Subject: [PATCH 0524/1790] added more resources section --- doc/source/quickstart.rst | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 33ddf5901..2a9e41e62 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -222,17 +222,23 @@ Compare staging area to head commit :end-before: # ![11.1-test_cloned_repo_object] .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [11.2-test_cloned_repo_object] - :end-before: # ![11.2-test_cloned_repo_object] + :language: python + :dedent: 8 + :start-after: # [11.2-test_cloned_repo_object] + :end-before: # ![11.2-test_cloned_repo_object] Compare commit to commit .. literalinclude:: ../../test/test_quick_doc.py - :language: python - :dedent: 8 - :start-after: # [11.3-test_cloned_repo_object] - :end-before: # ![11.3-test_cloned_repo_object] + :language: python + :dedent: 8 + :start-after: # [11.3-test_cloned_repo_object] + :end-before: # ![11.3-test_cloned_repo_object] + +More Resources +**************** +Remember, this is just the beginning! There's a lot more you can achieve with GitPython in your development workflow. +To explore further possibilities and discover advanced features, check out the full :ref:`GitPython tutorial ` +and the :ref:`API Reference `. Happy coding! From cf3a899ebd498bd8053bc17dab1ff4c36edc005e Mon Sep 17 00:00:00 2001 From: LeoDaCoda Date: Wed, 19 Jul 2023 13:50:25 -0400 Subject: [PATCH 0525/1790] typo --- doc/source/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 2a9e41e62..c5930eb8a 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -148,7 +148,7 @@ List of commits associated with a file Printing text files #################### -Lets print the latest version of ` dir1/file2.txt` +Lets print the latest version of `/dir1/file2.txt` .. literalinclude:: ../../test/test_quick_doc.py :language: python From 31ac93b2f38f2410e41bf90ad28dff31e79b114e Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 16:25:10 +0200 Subject: [PATCH 0526/1790] Do not typecheck submodule It has too many errors. Fixing them should be done in the separate project. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0d5ebf012..4d2014afb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ implicit_reexport = true # strict = true # TODO: remove when 'gitdb' is fully annotated +exclude = ["^git/ext/gitdb"] [[tool.mypy.overrides]] module = "gitdb.*" ignore_missing_imports = true From b55cf65cc96740c6128987ab0c07b43112bdfe31 Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 16:34:39 +0200 Subject: [PATCH 0527/1790] Define supported version for mypy --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4d2014afb..57988372a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ filterwarnings = 'ignore::DeprecationWarning' # filterwarnings ignore::WarningType # ignores those warnings [tool.mypy] +python_version = "3.7" disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true From 76394d42ce2a33b4db71fd64763c1e9dae136747 Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 16:39:32 +0200 Subject: [PATCH 0528/1790] Ignore remaining [unreachable] type errors --- git/__init__.py | 2 +- git/config.py | 4 ++-- git/repo/base.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index cd6602bf0..6196a42d7 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -76,7 +76,7 @@ def refresh(path: Optional[PathLike] = None) -> None: if not Git.refresh(path=path): return if not FetchInfo.refresh(): - return + return # type: ignore [unreachable] GIT_OK = True diff --git a/git/config.py b/git/config.py index e05a297af..caf1f6241 100644 --- a/git/config.py +++ b/git/config.py @@ -265,8 +265,8 @@ def get_config_path(config_level: Lit_config_levels) -> str: raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path") else: # Should not reach here. Will raise ValueError if does. Static typing will warn missing elifs - assert_never( - config_level, # type: ignore[unreachable] + assert_never( # type: ignore[unreachable] + config_level, ValueError(f"Invalid configuration level: {config_level!r}"), ) diff --git a/git/repo/base.py b/git/repo/base.py index 1fa98d8c7..4bfead46f 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -549,8 +549,8 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa return osp.normpath(osp.join(repo_dir, "config")) else: - assert_never( - config_level, # type:ignore[unreachable] + assert_never( # type:ignore[unreachable] + config_level, ValueError(f"Invalid configuration level: {config_level!r}"), ) From c6dab191d1f96373aaae5c6c117f13c1006631de Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 16:49:02 +0200 Subject: [PATCH 0529/1790] Allow explicit casting even when slightly redundant --- git/cmd.py | 2 +- git/objects/commit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index dfce9024d..5b0b6b816 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -154,7 +154,7 @@ def pump_stream( p_stdout = process.proc.stdout if process.proc else None p_stderr = process.proc.stderr if process.proc else None else: - process = cast(Popen, process) + process = cast(Popen, process) # type: ignore [redundant-cast] cmdline = getattr(process, "args", "") p_stdout = process.stdout p_stderr = process.stderr diff --git a/git/objects/commit.py b/git/objects/commit.py index 138db0afe..6cca65c1f 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -460,7 +460,7 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, if proc_or_stream.stdout is not None: stream = proc_or_stream.stdout elif hasattr(proc_or_stream, "readline"): - proc_or_stream = cast(IO, proc_or_stream) + proc_or_stream = cast(IO, proc_or_stream) # type: ignore [redundant-cast] stream = proc_or_stream readline = stream.readline From 6035db092decf72e6a01e175d044f0343818b51c Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 16:51:50 +0200 Subject: [PATCH 0530/1790] Run black and exclude submodule --- README.md | 2 ++ git/cmd.py | 7 ++----- git/config.py | 3 +-- git/diff.py | 3 +-- git/exc.py | 2 -- git/index/fun.py | 1 - git/objects/commit.py | 4 +--- git/objects/fun.py | 1 - git/objects/util.py | 2 +- git/remote.py | 1 - git/repo/base.py | 3 +-- git/util.py | 2 -- pyproject.toml | 1 + 13 files changed, 10 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 676d2c6d6..30c54b57f 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ To typecheck, run: `mypy -p git` To test, run: `pytest` +For automatic code formatting run: `black git` + Configuration for flake8 is in the ./.flake8 file. Configurations for mypy, pytest and coverage.py are in ./pyproject.toml. diff --git a/git/cmd.py b/git/cmd.py index 5b0b6b816..84d888494 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -122,6 +122,7 @@ def handle_process_output( To specify a timeout in seconds for the git command, after which the process should be killed. """ + # Use 2 "pump" threads and wait for both to finish. def pump_stream( cmdline: List[str], @@ -488,10 +489,7 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> """ # Options can be of the form `foo` or `--foo bar` `--foo=bar`, # so we need to check if they start with "--foo" or if they are equal to "foo". - bare_unsafe_options = [ - option.lstrip("-") - for option in unsafe_options - ] + bare_unsafe_options = [option.lstrip("-") for option in unsafe_options] for option in options: for unsafe_option, bare_option in zip(unsafe_options, bare_unsafe_options): if option.startswith(unsafe_option) or option == bare_option: @@ -1194,7 +1192,6 @@ def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any @classmethod def _unpack_args(cls, arg_list: Sequence[str]) -> List[str]: - outlist = [] if isinstance(arg_list, (list, tuple)): for arg in arg_list: diff --git a/git/config.py b/git/config.py index caf1f6241..1973111eb 100644 --- a/git/config.py +++ b/git/config.py @@ -248,7 +248,6 @@ def items_all(self) -> List[Tuple[str, List[_T]]]: def get_config_path(config_level: Lit_config_levels) -> str: - # we do not support an absolute path of the gitconfig on windows , # use the global config instead if is_win and config_level == "system": @@ -655,7 +654,7 @@ def write_section(name: str, section_dict: _OMD) -> None: values: Sequence[str] # runtime only gets str in tests, but should be whatever _OMD stores v: str - for (key, values) in section_dict.items_all(): + for key, values in section_dict.items_all(): if key == "__name__": continue diff --git a/git/diff.py b/git/diff.py index c1a5bd26f..1424ff3ad 100644 --- a/git/diff.py +++ b/git/diff.py @@ -145,7 +145,7 @@ def diff( args.append("--full-index") # get full index paths, not only filenames # remove default '-M' arg (check for renames) if user is overriding it - if not any(x in kwargs for x in ('find_renames', 'no_renames', 'M')): + if not any(x in kwargs for x in ("find_renames", "no_renames", "M")): args.append("-M") if create_patch: @@ -338,7 +338,6 @@ def __init__( change_type: Optional[Lit_change_type], score: Optional[int], ) -> None: - assert a_rawpath is None or isinstance(a_rawpath, bytes) assert b_rawpath is None or isinstance(b_rawpath, bytes) self.a_rawpath = a_rawpath diff --git a/git/exc.py b/git/exc.py index 9b69a5889..775528bf6 100644 --- a/git/exc.py +++ b/git/exc.py @@ -139,7 +139,6 @@ def __init__( valid_files: Sequence[PathLike], failed_reasons: List[str], ) -> None: - Exception.__init__(self, message) self.failed_files = failed_files self.failed_reasons = failed_reasons @@ -170,7 +169,6 @@ def __init__( stderr: Union[bytes, str, None] = None, stdout: Union[bytes, str, None] = None, ) -> None: - super(HookExecutionError, self).__init__(command, status, stderr, stdout) self._msg = "Hook('%s') failed%s" diff --git a/git/index/fun.py b/git/index/fun.py index d0925ed51..3dc5e96d2 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -394,7 +394,6 @@ def aggressive_tree_merge(odb: "GitCmdObjectDB", tree_shas: Sequence[bytes]) -> out.append(_tree_entry_to_baseindexentry(theirs, 0)) # END handle modification else: - if ours[0] != base[0] or ours[1] != base[1]: # they deleted it, we changed it, conflict out.append(_tree_entry_to_baseindexentry(base, 1)) diff --git a/git/objects/commit.py b/git/objects/commit.py index 6cca65c1f..6db3ea0f3 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -345,9 +345,7 @@ def trailers(self) -> Dict[str, str]: Dictionary containing whitespace stripped trailer information. Only contains the latest instance of each trailer key. """ - return { - k: v[0] for k, v in self.trailers_dict.items() - } + return {k: v[0] for k, v in self.trailers_dict.items()} @property def trailers_list(self) -> List[Tuple[str, str]]: diff --git a/git/objects/fun.py b/git/objects/fun.py index e91403a8b..043eec721 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -190,7 +190,6 @@ def traverse_trees_recursive( # is a tree. If the match is a non-tree item, put it into the result. # Processed items will be set None for ti, tree_data in enumerate(trees_data): - for ii, item in enumerate(tree_data): if not item: continue diff --git a/git/objects/util.py b/git/objects/util.py index af279154c..d72c04d17 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -143,7 +143,7 @@ def utctz_to_altz(utctz: str) -> int: :param utctz: git utc timezone string, i.e. +0200 """ int_utctz = int(utctz) - seconds = ((abs(int_utctz) // 100) * 3600 + (abs(int_utctz) % 100) * 60) + seconds = (abs(int_utctz) // 100) * 3600 + (abs(int_utctz) % 100) * 60 return seconds if int_utctz < 0 else -seconds diff --git a/git/remote.py b/git/remote.py index 5886a69f0..95a2b8ac6 100644 --- a/git/remote.py +++ b/git/remote.py @@ -826,7 +826,6 @@ def _get_fetch_info_from_stderr( progress: Union[Callable[..., Any], RemoteProgress, None], kill_after_timeout: Union[None, float] = None, ) -> IterableList["FetchInfo"]: - progress = to_progress_instance(progress) # skip first line as it is some remote info we are not interested in diff --git a/git/repo/base.py b/git/repo/base.py index 4bfead46f..723613c6f 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -498,7 +498,7 @@ def delete_head(self, *heads: "Union[str, Head]", **kwargs: Any) -> None: def create_tag( self, path: PathLike, - ref: Union[str, 'SymbolicReference'] = "HEAD", + ref: Union[str, "SymbolicReference"] = "HEAD", message: Optional[str] = None, force: bool = False, **kwargs: Any, @@ -548,7 +548,6 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa else: return osp.normpath(osp.join(repo_dir, "config")) else: - assert_never( # type:ignore[unreachable] config_level, ValueError(f"Invalid configuration level: {config_level!r}"), diff --git a/git/util.py b/git/util.py index 30028b1c2..5bfe11cd8 100644 --- a/git/util.py +++ b/git/util.py @@ -1083,7 +1083,6 @@ def __getattr__(self, attr: str) -> T_IterableObj: return list.__getattribute__(self, attr) def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore - assert isinstance(index, (int, str, slice)), "Index of IterableList should be an int or str" if isinstance(index, int): @@ -1098,7 +1097,6 @@ def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_Iterabl # END handle getattr def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: - assert isinstance(index, (int, str)), "Index of IterableList should be an int or str" delindex = cast(int, index) diff --git a/pyproject.toml b/pyproject.toml index 57988372a..32c9d4a26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,3 +45,4 @@ omit = ["*/git/ext/*"] [tool.black] line-length = 120 target-version = ['py37'] +exclude = "git/ext/gitdb" From 3908e79baf27b3d65265ca75db216f9368748351 Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 16:54:10 +0200 Subject: [PATCH 0531/1790] Add missing type annotation --- git/index/fun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/fun.py b/git/index/fun.py index 3dc5e96d2..4a2f3cb6d 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -76,7 +76,7 @@ def hook_path(name: str, git_dir: PathLike) -> str: return osp.join(git_dir, "hooks", name) -def _has_file_extension(path): +def _has_file_extension(path: str) -> str: return osp.splitext(path)[1] From f01ee4f8d0b83f06fc7ba5458ac896ac3b81184a Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Thu, 20 Jul 2023 17:21:41 +0200 Subject: [PATCH 0532/1790] Apply straight-forward typing fixes --- git/cmd.py | 2 +- git/index/base.py | 2 +- git/index/fun.py | 8 ++++---- git/objects/util.py | 2 +- git/util.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 84d888494..3d170facd 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -211,7 +211,7 @@ def dashify(string: str) -> str: return string.replace("_", "-") -def slots_to_dict(self: object, exclude: Sequence[str] = ()) -> Dict[str, Any]: +def slots_to_dict(self: "Git", exclude: Sequence[str] = ()) -> Dict[str, Any]: return {s: getattr(self, s) for s in self.__slots__ if s not in exclude} diff --git a/git/index/base.py b/git/index/base.py index dd8f9aa2e..193baf3ad 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -656,7 +656,7 @@ def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry def _entries_for_paths( self, paths: List[str], - path_rewriter: Callable, + path_rewriter: Union[Callable, None], fprogress: Callable, entries: List[BaseIndexEntry], ) -> List[BaseIndexEntry]: diff --git a/git/index/fun.py b/git/index/fun.py index 4a2f3cb6d..b50f1f465 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -102,7 +102,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: relative_hp = Path(hp).relative_to(index.repo.working_dir).as_posix() cmd = ["bash.exe", relative_hp] - cmd = subprocess.Popen( + process = subprocess.Popen( cmd + list(args), env=env, stdout=subprocess.PIPE, @@ -116,13 +116,13 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: else: stdout_list: List[str] = [] stderr_list: List[str] = [] - handle_process_output(cmd, stdout_list.append, stderr_list.append, finalize_process) + handle_process_output(process, stdout_list.append, stderr_list.append, finalize_process) stdout = "".join(stdout_list) stderr = "".join(stderr_list) - if cmd.returncode != 0: + if process.returncode != 0: stdout = force_text(stdout, defenc) stderr = force_text(stderr, defenc) - raise HookExecutionError(hp, cmd.returncode, stderr, stdout) + raise HookExecutionError(hp, process.returncode, stderr, stdout) # end handle return code diff --git a/git/objects/util.py b/git/objects/util.py index d72c04d17..56938507e 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -147,7 +147,7 @@ def utctz_to_altz(utctz: str) -> int: return seconds if int_utctz < 0 else -seconds -def altz_to_utctz_str(altz: int) -> str: +def altz_to_utctz_str(altz: float) -> str: """Convert a timezone offset west of UTC in seconds into a git timezone offset string :param altz: timezone offset in seconds west of UTC diff --git a/git/util.py b/git/util.py index 5bfe11cd8..0ef8bdeb7 100644 --- a/git/util.py +++ b/git/util.py @@ -1049,7 +1049,7 @@ class IterableList(List[T_IterableObj]): __slots__ = ("_id_attr", "_prefix") - def __new__(cls, id_attr: str, prefix: str = "") -> "IterableList[IterableObj]": + def __new__(cls, id_attr: str, prefix: str = "") -> "IterableList[T_IterableObj]": return super(IterableList, cls).__new__(cls) def __init__(self, id_attr: str, prefix: str = "") -> None: From 41ecc6a4f80ecaa07ceac59861820c4b88dd5d1e Mon Sep 17 00:00:00 2001 From: Bodo Graumann Date: Fri, 21 Jul 2023 09:21:46 +0200 Subject: [PATCH 0533/1790] Disable merge_includes in config writers --- git/repo/base.py | 2 +- test/test_refs.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index 723613c6f..ab2026549 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -600,7 +600,7 @@ def config_writer(self, config_level: Lit_config_levels = "repository") -> GitCo system = system wide configuration file global = user level configuration file repository = configuration file for this repository only""" - return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self) + return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self, merge_includes=False) def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: """The Commit object for the specified revision diff --git a/test/test_refs.py b/test/test_refs.py index 5bb83100e..4c421767e 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -15,6 +15,7 @@ SymbolicReference, GitCommandError, RefLog, + GitConfigParser, ) from git.objects.tag import TagObject from test.lib import TestBase, with_rw_repo @@ -172,6 +173,26 @@ def test_heads(self, rwrepo): assert log[0].oldhexsha == pcommit.NULL_HEX_SHA assert log[0].newhexsha == pcommit.hexsha + @with_rw_repo("HEAD", bare=False) + def test_set_tracking_branch_with_import(self, rwrepo): + # prepare included config file + included_config = osp.join(rwrepo.git_dir, "config.include") + with GitConfigParser(included_config, read_only=False) as writer: + writer.set_value("test", "value", "test") + assert osp.exists(included_config) + + with rwrepo.config_writer() as writer: + writer.set_value("include", "path", included_config) + + for head in rwrepo.heads: + head.set_tracking_branch(None) + assert head.tracking_branch() is None + remote_ref = rwrepo.remotes[0].refs[0] + assert head.set_tracking_branch(remote_ref) is head + assert head.tracking_branch() == remote_ref + head.set_tracking_branch(None) + assert head.tracking_branch() is None + def test_refs(self): types_found = set() for ref in self.rorepo.refs: From 186c1ae12be1bb76087dd4fa53a1ac0979c8aa9f Mon Sep 17 00:00:00 2001 From: Patrick Hagemeister Date: Thu, 27 Jul 2023 06:24:20 +0000 Subject: [PATCH 0534/1790] Creating a lock now uses python built-in "open()" method to work around docker virtiofs issue --- git/util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/git/util.py b/git/util.py index 0ef8bdeb7..a3748f0fe 100644 --- a/git/util.py +++ b/git/util.py @@ -935,11 +935,7 @@ def _obtain_lock_or_raise(self) -> None: ) try: - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if is_win: - flags |= os.O_SHORT_LIVED - fd = os.open(lock_file, flags, 0) - os.close(fd) + open(lock_file, mode='w', closefd=True) except OSError as e: raise IOError(str(e)) from e From 9f74c05b7d0f224bb170ad77675d0d2b0ff82a0d Mon Sep 17 00:00:00 2001 From: Lydia Date: Sun, 27 Aug 2023 18:28:59 +0200 Subject: [PATCH 0535/1790] feat: full typing for "progress" parameter This commit also fix a few inconsistency, most especially: - How mypy/Pylance are checking for if-statements - Tools complaining about not subscriptable class - Exporting imports --- git/repo/base.py | 5 +++-- git/types.py | 22 +++++++++++----------- requirements-dev.txt | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index ab2026549..113fca459 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -60,6 +60,7 @@ PathLike, Lit_config_levels, Commit_ish, + CallableProgress, Tree_ish, assert_never, ) @@ -1258,7 +1259,7 @@ def _clone( def clone( self, path: PathLike, - progress: Optional[Callable] = None, + progress: Optional[CallableProgress] = None, multi_options: Optional[List[str]] = None, allow_unsafe_protocols: bool = False, allow_unsafe_options: bool = False, @@ -1297,7 +1298,7 @@ def clone_from( cls, url: PathLike, to_path: PathLike, - progress: Optional[Callable] = None, + progress: CallableProgress = None, env: Optional[Mapping[str, str]] = None, multi_options: Optional[List[str]] = None, allow_unsafe_protocols: bool = False, diff --git a/git/types.py b/git/types.py index 9064ecbf9..9f8621721 100644 --- a/git/types.py +++ b/git/types.py @@ -8,42 +8,39 @@ from typing import ( Dict, NoReturn, - Sequence, + Sequence as Sequence, Tuple, Union, Any, + Optional, + Callable, TYPE_CHECKING, TypeVar, ) # noqa: F401 -if sys.version_info[:2] >= (3, 8): +if sys.version_info >= (3, 8): from typing import ( Literal, - SupportsIndex, TypedDict, Protocol, + SupportsIndex as SupportsIndex, runtime_checkable, ) # noqa: F401 else: from typing_extensions import ( Literal, - SupportsIndex, # noqa: F401 + SupportsIndex as SupportsIndex, TypedDict, Protocol, runtime_checkable, ) # noqa: F401 -# if sys.version_info[:2] >= (3, 10): +# if sys.version_info >= (3, 10): # from typing import TypeGuard # noqa: F401 # else: # from typing_extensions import TypeGuard # noqa: F401 - -if sys.version_info[:2] < (3, 9): - PathLike = Union[str, os.PathLike] -else: - # os.PathLike only becomes subscriptable from Python 3.9 onwards - PathLike = Union[str, os.PathLike[str]] +PathLike = Union[str, "os.PathLike[str]"] if TYPE_CHECKING: from git.repo import Repo @@ -62,6 +59,9 @@ Lit_config_levels = Literal["system", "global", "user", "repository"] +# Progress parameter type alias ----------------------------------------- + +CallableProgress = Optional[Callable[[int, Union[str, float], Union[str, float, None], str], None]] # def is_config_level(inp: str) -> TypeGuard[Lit_config_levels]: # # return inp in get_args(Lit_config_level) # only py >= 3.8 diff --git a/requirements-dev.txt b/requirements-dev.txt index bacde3498..946b4c94f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,4 +10,4 @@ pytest-icdiff # pytest-profiling -tox \ No newline at end of file +tox From 6029211d729a0dd81e08fcc9c1a3ab7fe9af85c9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 30 Aug 2023 09:32:11 -0400 Subject: [PATCH 0536/1790] Fix CVE-2023-40590 This fixes the path search bug where the current directory is included on Windows, by setting NoDefaultCurrentDirectoryInExePath for the caller. (Setting for the callee env would not work.) This sets it only on Windows, only for the duration of the Popen call, and then automatically unsets it or restores its old value. NoDefaultCurrentDirectoryInExePath is documented at: https://learn.microsoft.com/en-us/windows/win32/api/processenv/nf-processenv-needcurrentdirectoryforexepathw It automatically affects the behavior of subprocess.Popen on Windows, due to the way Popen uses the Windows API. (In contrast, it does not, at least currently on CPython, affect the behavior of shutil.which. But shutil.which is not being used to find git.exe.) --- git/cmd.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 3d170facd..3665eb029 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,7 +5,7 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php from __future__ import annotations import re -from contextlib import contextmanager +import contextlib import io import logging import os @@ -14,6 +14,7 @@ import subprocess import threading from textwrap import dedent +import unittest.mock from git.compat import ( defenc, @@ -963,8 +964,11 @@ def execute( redacted_command, '"kill_after_timeout" feature is not supported on Windows.', ) + # Only search PATH, not CWD. This must be in the *caller* environment. The "1" can be any value. + patch_caller_env = unittest.mock.patch.dict(os.environ, {"NoDefaultCurrentDirectoryInExePath": "1"}) else: cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable + patch_caller_env = contextlib.nullcontext() # end handle stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") @@ -980,21 +984,21 @@ def execute( istream_ok, ) try: - proc = Popen( - command, - env=env, - cwd=cwd, - bufsize=-1, - stdin=istream or DEVNULL, - stderr=PIPE, - stdout=stdout_sink, - shell=shell is not None and shell or self.USE_SHELL, - close_fds=is_posix, # unsupported on windows - universal_newlines=universal_newlines, - creationflags=PROC_CREATIONFLAGS, - **subprocess_kwargs, - ) - + with patch_caller_env: + proc = Popen( + command, + env=env, + cwd=cwd, + bufsize=-1, + stdin=istream or DEVNULL, + stderr=PIPE, + stdout=stdout_sink, + shell=shell is not None and shell or self.USE_SHELL, + close_fds=is_posix, # unsupported on windows + universal_newlines=universal_newlines, + creationflags=PROC_CREATIONFLAGS, + **subprocess_kwargs, + ) except cmd_not_found_exception as err: raise GitCommandNotFound(redacted_command, err) from err else: @@ -1144,7 +1148,7 @@ def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: del self._environment[key] return old_env - @contextmanager + @contextlib.contextmanager def custom_environment(self, **kwargs: Any) -> Iterator[None]: """ A context manager around the above ``update_environment`` method to restore the From 94e0fb0794b88b78ceed94ff18ee7d68587d890d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 30 Aug 2023 11:53:29 -0400 Subject: [PATCH 0537/1790] Add a unit test for CVE-2023-40590 This adds test_it_executes_git_not_from_cwd to verify that the execute method does not use "git.exe" in the current directory on Windows, nor "git" in the current directory on Unix-like systems, when those files are executable. It adds a _chdir helper context manager to support this, because contextlib.chdir is only available on Python 3.11 and later. --- test/test_git.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/test_git.py b/test/test_git.py index c5d871f08..01572dc24 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -4,10 +4,12 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +import contextlib import os +import shutil import subprocess import sys -from tempfile import TemporaryFile +from tempfile import TemporaryDirectory, TemporaryFile from unittest import mock from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd @@ -20,6 +22,17 @@ from git.compat import is_win +@contextlib.contextmanager +def _chdir(new_dir): + """Context manager to temporarily change directory. Not reentrant.""" + old_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(old_dir) + + class TestGit(TestBase): @classmethod def setUpClass(cls): @@ -75,6 +88,23 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): def test_it_executes_git_to_shell_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") + def test_it_executes_git_not_from_cwd(self): + with TemporaryDirectory() as tmpdir: + if is_win: + # Copy an actual binary executable that is not git. + other_exe_path = os.path.join(os.getenv("WINDIR"), "system32", "hostname.exe") + impostor_path = os.path.join(tmpdir, "git.exe") + shutil.copy(other_exe_path, impostor_path) + else: + # Create a shell script that doesn't do anything. + impostor_path = os.path.join(tmpdir, "git") + with open(impostor_path, mode="w", encoding="utf-8") as file: + print("#!/bin/sh", file=file) + os.chmod(impostor_path, 0o755) + + with _chdir(tmpdir): + self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") + def test_it_accepts_stdin(self): filename = fixture_path("cat_file_blob") with open(filename, "r") as fh: From 7611cd909b890b971d23bce3bd4244ad1c381f22 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 30 Aug 2023 12:34:22 -0400 Subject: [PATCH 0538/1790] Don't check form of version number This changes the regex in test_it_executes_git_not_from_cwd so that (unlike test_it_executes_git_to_shell_and_returns_result) it only checks that the output starts with the words "git version", and not the form of whatever follows those words. --- test/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_git.py b/test/test_git.py index 01572dc24..540ea9f41 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -103,7 +103,7 @@ def test_it_executes_git_not_from_cwd(self): os.chmod(impostor_path, 0o755) with _chdir(tmpdir): - self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") + self.assertRegex(self.git.execute(["git", "version"]), r"^git version\b") def test_it_accepts_stdin(self): filename = fixture_path("cat_file_blob") From 70924c4265c2d3629d978dd7bfc9ab1678d91e7d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 1 Sep 2023 08:11:22 +0200 Subject: [PATCH 0539/1790] Skip now permanently failing test with note on how to fix it --- test/test_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_repo.py b/test/test_repo.py index 5c66aeeb1..08ed13a00 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -13,7 +13,7 @@ import pickle import sys import tempfile -from unittest import mock, skipIf, SkipTest +from unittest import mock, skipIf, SkipTest, skip import pytest @@ -251,6 +251,7 @@ def test_clone_from_with_path_contains_unicode(self): self.fail("Raised UnicodeEncodeError") @with_rw_directory + @skip("the referenced repository was removed, and one needs to setup a new password controlled repo under the orgs control") def test_leaking_password_in_clone_logs(self, rw_dir): password = "fakepassword1234" try: From 993f04588aa362fdce7c7f2f0848b5daedd8cb72 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 1 Sep 2023 08:12:54 +0200 Subject: [PATCH 0540/1790] prepare for next release --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 381c34a62..f8d874e2b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.32 +3.1.33 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 3bc02e770..45062ac13 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.33 +====== + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/63?closed=1 + 3.1.32 ====== From f882cd8422fbb2517eebbf45824eb07951b948f3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 1 Sep 2023 08:48:02 +0200 Subject: [PATCH 0541/1790] update instructions for how to create a release --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 30c54b57f..1743bd3d2 100644 --- a/README.md +++ b/README.md @@ -144,9 +144,7 @@ Please have a look at the [contributions file][contributing]. - Run `git tag -s ` to tag the version in Git - Run `make release` - Close the milestone mentioned in the _changelog_ and create a new one. _Do not reuse milestones by renaming them_. -- set the upcoming version in the `VERSION` file, usually be - incrementing the patch level, and possibly by appending `-dev`. Probably you - want to `git push` once more. +- Got to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. ### How to verify a release (DEPRECATED) From 3e829eb516a60212bae81a6549361be4748e22d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Sat, 2 Sep 2023 14:21:03 +0545 Subject: [PATCH 0542/1790] util: close lockfile after opening successfully Otherwise, this will leak file handles and can be a problem in Windows. Also, `closefd=true` is the default here, so need to pass it explicitly. Regression from #1619. I noticed after [our tests started raising `ResourceWarning`][1]. ```python Traceback (most recent call last): File "/opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/git/util.py", line 938, in _obtain_lock_or_raise open(lock_file, mode='w', closefd=True) ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/pytest-of-runner/pytest-0/popen-gw0/external0/project.git/.git/config.lock' mode='w' encoding='UTF-8'> ``` [1]: https://github.com/iterative/dvc/actions/runs/6055520480/job/16434544764#step:6:869 --- git/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index a3748f0fe..f6dedf0f2 100644 --- a/git/util.py +++ b/git/util.py @@ -935,7 +935,8 @@ def _obtain_lock_or_raise(self) -> None: ) try: - open(lock_file, mode='w', closefd=True) + with open(lock_file, mode='w'): + pass except OSError as e: raise IOError(str(e)) from e From 2a2ae776825f249a3bb7efd9b08650486226b027 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 2 Sep 2023 11:29:43 +0200 Subject: [PATCH 0543/1790] prepare patch release --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f8d874e2b..e03213a2b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.33 +3.1.34 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 45062ac13..38e673f3f 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.34 +====== + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/64?closed=1 + 3.1.33 ====== From b32e01e33d5e4913b349d2ccfc08e3981d8c9621 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 13:50:12 +0000 Subject: [PATCH 0544/1790] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index a1ecb6785..6ba63f019 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Force LF line endings run: git config --global core.autocrlf input - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 9999 - uses: cygwin/cygwin-install-action@v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c78a4053a..5e79664a8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 2d95e6ffa..68988f2a7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,7 +18,7 @@ jobs: python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 9999 - name: Set up Python ${{ matrix.python-version }} From d5e763ea76d28cab14e16a4bc23572986fbf1797 Mon Sep 17 00:00:00 2001 From: "Wenhan Zhu (Cosmos)" Date: Tue, 5 Sep 2023 19:30:17 -0400 Subject: [PATCH 0545/1790] Fix 'Tree' object has no attribute '_name' when submodule path is normal path --- git/objects/submodule/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 7db64d705..0d20305c6 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1402,6 +1402,10 @@ def iter_items( # END handle keyerror # END handle critical error + # Make sure we are looking at a submodule object + if type(sm) != git.objects.submodule.base.Submodule: + continue + # fill in remaining info - saves time as it doesn't have to be parsed again sm._name = n if pc != repo.commit(): From 64ebb9fcdfbe48d5d61141a557691fd91f1e88d6 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Tue, 5 Sep 2023 09:51:50 +0200 Subject: [PATCH 0546/1790] Fix CVE-2023-41040 This change adds a check during reference resolving to see if it contains an up-level reference ('..'). If it does, it raises an exception. This fixes CVE-2023-41040, which allows an attacker to access files outside the repository's directory. --- git/refs/symbolic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 33c3bf15b..5c293aa7b 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -168,6 +168,8 @@ def _get_ref_info_helper( """Return: (str(sha), str(target_ref_path)) if available, the sha the file at rela_path points to, or None. target_ref_path is the reference we point to, or None""" + if ".." in str(ref_path): + raise ValueError(f"Invalid reference '{ref_path}'") tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: From 65b8c6a2ccacdf26e751cd3bc3c5a7c9e5796b56 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Tue, 5 Sep 2023 13:49:38 +0200 Subject: [PATCH 0547/1790] Add test for CVE-2023-41040 --- test/test_refs.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test_refs.py b/test/test_refs.py index 4c421767e..e7526c3b2 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -5,6 +5,7 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php from itertools import chain +from pathlib import Path from git import ( Reference, @@ -20,9 +21,11 @@ from git.objects.tag import TagObject from test.lib import TestBase, with_rw_repo from git.util import Actor +from gitdb.exc import BadName import git.refs as refs import os.path as osp +import tempfile class TestRefs(TestBase): @@ -616,3 +619,15 @@ def test_dereference_recursive(self): def test_reflog(self): assert isinstance(self.rorepo.heads.master.log(), RefLog) + + def test_refs_outside_repo(self): + # Create a file containing a valid reference outside the repository. Attempting + # to access it should raise an exception, due to it containing a parent directory + # reference ('..'). This tests for CVE-2023-41040. + git_dir = Path(self.rorepo.git_dir) + repo_parent_dir = git_dir.parent.parent + with tempfile.NamedTemporaryFile(dir=repo_parent_dir) as ref_file: + ref_file.write(b"91b464cd624fe22fbf54ea22b85a7e5cca507cfe") + ref_file.flush() + ref_file_name = Path(ref_file.name).name + self.assertRaises(BadName, self.rorepo.commit, f"../../{ref_file_name}") From 537af83c344420994a6a34dd18623f132398d062 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Sep 2023 17:44:25 -0400 Subject: [PATCH 0548/1790] Only set safe.directory on Cygwin (which needs it) This stops setting the current directory as an explicit safe directory on CI for non-Windows systems, where this is not needed because the repository has the ownership Git expects. The step name is updated accordingly to reflect its now narrower purpose. This also adds shell quoting to $(pwd) in the Cygwin workflow. In practice, on CI, the path is very unlikely to contain whitespace, but double-quoting $ expansions on which splitting and globbing are unwanted is more robust and better expresses intent. This also has the benefit that users who use the CI workflows as a guide to commands they run locally, where on Windows they may very well have spaces somewhere in this absolute path, will use a correct command. --- .github/workflows/cygwin-test.yml | 6 +++--- .github/workflows/pythonpackage.yml | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 6ba63f019..618bec405 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -12,7 +12,7 @@ jobs: SHELLOPTS: igncr TMP: "/tmp" TEMP: "/tmp" - + steps: - name: Force LF line endings run: git config --global core.autocrlf input @@ -24,8 +24,8 @@ jobs: packages: python39 python39-pip python39-virtualenv git - name: Tell git to trust this repo shell: bash.exe -eo pipefail -o igncr "{0}" - run: | - /usr/bin/git config --global --add safe.directory $(pwd) + run: | + /usr/bin/git config --global --add safe.directory "$(pwd)" /usr/bin/git config --global protocol.file.allow always - name: Install dependencies and prepare tests shell: bash.exe -eo pipefail -o igncr "{0}" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 68988f2a7..bc50cab29 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -52,9 +52,8 @@ jobs: set -x mypy -p git - - name: Tell git to trust this repo - run: | - /usr/bin/git config --global --add safe.directory $(pwd) + - name: Tell git to allow file protocol even for submodules + run: | /usr/bin/git config --global protocol.file.allow always - name: Test with pytest From 92d9ae22132c97f764f0108da30062c24392cc2b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Sep 2023 18:02:20 -0400 Subject: [PATCH 0549/1790] Use env vars on CI to set protocol.file.allow Instead of setting a global git configuration. This makes no significant difference for security on CI, but it is an iterative step toward a more specific way of setting them that will apply on CI and locally and require less configuration. In addition, this shows an approach more similar to what users who do not want to carefully review the security impact of changing the global setting can use locally (and which is more secure). --- .github/workflows/cygwin-test.yml | 5 ++++- .github/workflows/pythonpackage.yml | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 618bec405..b612ca94a 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -26,7 +26,6 @@ jobs: shell: bash.exe -eo pipefail -o igncr "{0}" run: | /usr/bin/git config --global --add safe.directory "$(pwd)" - /usr/bin/git config --global protocol.file.allow always - name: Install dependencies and prepare tests shell: bash.exe -eo pipefail -o igncr "{0}" run: | @@ -47,4 +46,8 @@ jobs: shell: bash.exe -eo pipefail -o igncr "{0}" run: | /usr/bin/python -m pytest + env: + GIT_CONFIG_COUNT: "1" + GIT_CONFIG_KEY_0: protocol.file.allow + GIT_CONFIG_VALUE_0: always continue-on-error: false diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index bc50cab29..207714810 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -52,14 +52,14 @@ jobs: set -x mypy -p git - - name: Tell git to allow file protocol even for submodules - run: | - /usr/bin/git config --global protocol.file.allow always - - name: Test with pytest run: | set -x pytest + env: + GIT_CONFIG_COUNT: "1" + GIT_CONFIG_KEY_0: protocol.file.allow + GIT_CONFIG_VALUE_0: always continue-on-error: false - name: Documentation From 4f594cd2cbf68caabb7d3f22397104f5aa1b49b7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Sep 2023 18:59:12 -0400 Subject: [PATCH 0550/1790] Set protocol.file.allow only in tests that need it Instead of setting environment variables just on CI and for the the entire pytest command, this has the two test cases that need protocol.file.allow to be set to "always" (instead of "user") set them, via a shared fixture, just while those tests are running. Both on CI and for local test runs, this makes it no longer necessary to set this in a global configuration or through environment variables, reducing the setup needed to run the tests. --- .github/workflows/cygwin-test.yml | 4 ---- .github/workflows/pythonpackage.yml | 4 ---- test/test_submodule.py | 22 +++++++++++++++++++++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index b612ca94a..808dc5608 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -46,8 +46,4 @@ jobs: shell: bash.exe -eo pipefail -o igncr "{0}" run: | /usr/bin/python -m pytest - env: - GIT_CONFIG_COUNT: "1" - GIT_CONFIG_KEY_0: protocol.file.allow - GIT_CONFIG_VALUE_0: always continue-on-error: false diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 207714810..a6af507d1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -56,10 +56,6 @@ jobs: run: | set -x pytest - env: - GIT_CONFIG_COUNT: "1" - GIT_CONFIG_KEY_0: protocol.file.allow - GIT_CONFIG_VALUE_0: always continue-on-error: false - name: Documentation diff --git a/test/test_submodule.py b/test/test_submodule.py index 982226411..a039faaf3 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +import contextlib import os import shutil import tempfile from pathlib import Path import sys -from unittest import skipIf +from unittest import mock, skipIf import pytest @@ -31,6 +32,23 @@ import os.path as osp +@contextlib.contextmanager +def _allow_file_protocol(): + """Temporarily set protocol.file.allow to always, using environment variables.""" + pair_index = int(os.getenv("GIT_CONFIG_COUNT", "0")) + + # This is recomputed each time the context is entered, for compatibility with + # existing GIT_CONFIG_* environment variables, even if changed in this process. + patcher = mock.patch.dict(os.environ, { + "GIT_CONFIG_COUNT": str(pair_index + 1), + f"GIT_CONFIG_KEY_{pair_index}": "protocol.file.allow", + f"GIT_CONFIG_VALUE_{pair_index}": "always", + }) + + with patcher: + yield + + class TestRootProgress(RootUpdateProgress): """Just prints messages, for now without checking the correctness of the states""" @@ -709,6 +727,7 @@ def test_add_empty_repo(self, rwdir): # end for each checkout mode @with_rw_directory + @_allow_file_protocol() def test_list_only_valid_submodules(self, rwdir): repo_path = osp.join(rwdir, "parent") repo = git.Repo.init(repo_path) @@ -737,6 +756,7 @@ def test_list_only_valid_submodules(self, rwdir): """, ) @with_rw_directory + @_allow_file_protocol() def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) parent.git.submodule("add", self._small_repo_url(), "module") From f6c326288a04d17907081b065c436332e60115de Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Sep 2023 23:24:34 +0000 Subject: [PATCH 0551/1790] Redesign new decorator to better separate concerns _allow_file_protocol was effectively a _patch_git_config fixture, being no no shorter, simpler, or clearer by hard-coding the specific name and value to patch. So this changes it to be that. As a secondary issue, it previously was called with no arguments, then that would be used as a decorator. That was unintutive and it was easy to omit the parentheses accidentally. This resolves that. --- test/test_submodule.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index a039faaf3..d906a5d5b 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -33,16 +33,16 @@ @contextlib.contextmanager -def _allow_file_protocol(): - """Temporarily set protocol.file.allow to always, using environment variables.""" +def _patch_git_config(name, value): + """Temporarily add a git config name-value pair, using environment variables.""" pair_index = int(os.getenv("GIT_CONFIG_COUNT", "0")) # This is recomputed each time the context is entered, for compatibility with # existing GIT_CONFIG_* environment variables, even if changed in this process. patcher = mock.patch.dict(os.environ, { "GIT_CONFIG_COUNT": str(pair_index + 1), - f"GIT_CONFIG_KEY_{pair_index}": "protocol.file.allow", - f"GIT_CONFIG_VALUE_{pair_index}": "always", + f"GIT_CONFIG_KEY_{pair_index}": name, + f"GIT_CONFIG_VALUE_{pair_index}": value, }) with patcher: @@ -727,7 +727,7 @@ def test_add_empty_repo(self, rwdir): # end for each checkout mode @with_rw_directory - @_allow_file_protocol() + @_patch_git_config("protocol.file.allow", "always") def test_list_only_valid_submodules(self, rwdir): repo_path = osp.join(rwdir, "parent") repo = git.Repo.init(repo_path) @@ -756,7 +756,7 @@ def test_list_only_valid_submodules(self, rwdir): """, ) @with_rw_directory - @_allow_file_protocol() + @_patch_git_config("protocol.file.allow", "always") def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) parent.git.submodule("add", self._small_repo_url(), "module") From d88372a11ac145d92013dcc64b7d21a5a6ad3a91 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Sep 2023 06:12:08 -0400 Subject: [PATCH 0552/1790] Add test for Windows env var upcasing regression --- test/test_git.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/test_git.py b/test/test_git.py index 540ea9f41..8ed9b64fe 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -10,7 +10,7 @@ import subprocess import sys from tempfile import TemporaryDirectory, TemporaryFile -from unittest import mock +from unittest import mock, skipUnless from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd from test.lib import TestBase, fixture_path @@ -105,6 +105,27 @@ def test_it_executes_git_not_from_cwd(self): with _chdir(tmpdir): self.assertRegex(self.git.execute(["git", "version"]), r"^git version\b") + @skipUnless(is_win, "The regression only affected Windows, and this test logic is OS-specific.") + def test_it_avoids_upcasing_unrelated_environment_variable_names(self): + old_name = "28f425ca_d5d8_4257_b013_8d63166c8158" + if old_name == old_name.upper(): + raise RuntimeError("test bug or strange locale: old_name invariant under upcasing") + os.putenv(old_name, "1") # It has to be done this lower-level way to set it lower-case. + + script_lines = [ + "import subprocess, git", + + # Importing git should be enough, but this really makes sure Git.execute is called. + f"repo = git.Repo({self.rorepo.working_dir!r})", + "git.Git(repo.working_dir).execute(['git', 'version'])", + + f"print(subprocess.check_output(['set', {old_name!r}], shell=True, text=True))", + ] + cmdline = [sys.executable, "-c", "\n".join(script_lines)] + pair_text = subprocess.check_output(cmdline, shell=False, text=True) + new_name = pair_text.split("=")[0] + self.assertEqual(new_name, old_name) + def test_it_accepts_stdin(self): filename = fixture_path("cat_file_blob") with open(filename, "r") as fh: From 7296e5c021450743e5fe824e94b830a73eebc4c8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Sep 2023 06:36:34 -0400 Subject: [PATCH 0553/1790] Make test helper script a file, for readability --- test/fixtures/env_case.py | 13 +++++++++++++ test/test_git.py | 14 +++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/env_case.py diff --git a/test/fixtures/env_case.py b/test/fixtures/env_case.py new file mode 100644 index 000000000..120e59289 --- /dev/null +++ b/test/fixtures/env_case.py @@ -0,0 +1,13 @@ +import subprocess +import sys + +import git + + +_, working_dir, env_var_name = sys.argv + +# Importing git should be enough, but this really makes sure Git.execute is called. +repo = git.Repo(working_dir) # Hold the reference. +git.Git(repo.working_dir).execute(["git", "version"]) + +print(subprocess.check_output(["set", env_var_name], shell=True, text=True)) diff --git a/test/test_git.py b/test/test_git.py index 8ed9b64fe..804cd22e4 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -112,16 +112,12 @@ def test_it_avoids_upcasing_unrelated_environment_variable_names(self): raise RuntimeError("test bug or strange locale: old_name invariant under upcasing") os.putenv(old_name, "1") # It has to be done this lower-level way to set it lower-case. - script_lines = [ - "import subprocess, git", - - # Importing git should be enough, but this really makes sure Git.execute is called. - f"repo = git.Repo({self.rorepo.working_dir!r})", - "git.Git(repo.working_dir).execute(['git', 'version'])", - - f"print(subprocess.check_output(['set', {old_name!r}], shell=True, text=True))", + cmdline = [ + sys.executable, + fixture_path("env_case.py"), + self.rorepo.working_dir, + old_name, ] - cmdline = [sys.executable, "-c", "\n".join(script_lines)] pair_text = subprocess.check_output(cmdline, shell=False, text=True) new_name = pair_text.split("=")[0] self.assertEqual(new_name, old_name) From c7fad20be5df0a86636459bf673ff9242a82e1fc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Sep 2023 04:32:36 -0400 Subject: [PATCH 0554/1790] Fix Windows env var upcasing regression This uses a simple hand-rolled context manager to patch the NoDefaultCurrentDirectoryInExePath variable, instead of unittest.mock.patch.dict. The latter set unrelated environment variables to the original (same) values via os.environ, and as a result, their names were all converted to upper-case on Windows. Because only environment variables that are actually set through os.environ have their names upcased, the only variable whose name should be upcased now is NoDefaultCurrentDirectoryInExePath, which should be fine (it has a single established use/meaning in Windows, where it's treated case-insensitively as environment variables in Windows *usually* are). --- git/cmd.py | 9 ++++----- git/util.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 3665eb029..d6f8f946a 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -14,7 +14,6 @@ import subprocess import threading from textwrap import dedent -import unittest.mock from git.compat import ( defenc, @@ -24,7 +23,7 @@ is_win, ) from git.exc import CommandError -from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present +from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present, patch_env from .exc import GitCommandError, GitCommandNotFound, UnsafeOptionError, UnsafeProtocolError from .util import ( @@ -965,10 +964,10 @@ def execute( '"kill_after_timeout" feature is not supported on Windows.', ) # Only search PATH, not CWD. This must be in the *caller* environment. The "1" can be any value. - patch_caller_env = unittest.mock.patch.dict(os.environ, {"NoDefaultCurrentDirectoryInExePath": "1"}) + maybe_patch_caller_env = patch_env("NoDefaultCurrentDirectoryInExePath", "1") else: cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable - patch_caller_env = contextlib.nullcontext() + maybe_patch_caller_env = contextlib.nullcontext() # end handle stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") @@ -984,7 +983,7 @@ def execute( istream_ok, ) try: - with patch_caller_env: + with maybe_patch_caller_env: proc = Popen( command, env=env, diff --git a/git/util.py b/git/util.py index f6dedf0f2..f80580cff 100644 --- a/git/util.py +++ b/git/util.py @@ -158,6 +158,20 @@ def cwd(new_dir: PathLike) -> Generator[PathLike, None, None]: os.chdir(old_dir) +@contextlib.contextmanager +def patch_env(name: str, value: str) -> Generator[None, None, None]: + """Context manager to temporarily patch an environment variable.""" + old_value = os.getenv(name) + os.environ[name] = value + try: + yield + finally: + if old_value is None: + del os.environ[name] + else: + os.environ[name] = old_value + + def rmtree(path: PathLike) -> None: """Remove the given recursively. @@ -935,7 +949,7 @@ def _obtain_lock_or_raise(self) -> None: ) try: - with open(lock_file, mode='w'): + with open(lock_file, mode="w"): pass except OSError as e: raise IOError(str(e)) from e From eebdb25ee6e88d8fce83ea0970bd08f5e5301f65 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Sep 2023 06:59:02 -0400 Subject: [PATCH 0555/1790] Eliminate duplication of git.util.cwd logic --- git/util.py | 1 + test/test_git.py | 16 ++-------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/git/util.py b/git/util.py index f80580cff..dee467dd3 100644 --- a/git/util.py +++ b/git/util.py @@ -150,6 +150,7 @@ def wrapper(self: "Remote", *args: Any, **kwargs: Any) -> T: @contextlib.contextmanager def cwd(new_dir: PathLike) -> Generator[PathLike, None, None]: + """Context manager to temporarily change directory. Not reentrant.""" old_dir = os.getcwd() os.chdir(new_dir) try: diff --git a/test/test_git.py b/test/test_git.py index 804cd22e4..f1d35a355 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -4,7 +4,6 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -import contextlib import os import shutil import subprocess @@ -15,24 +14,13 @@ from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd from test.lib import TestBase, fixture_path from test.lib import with_rw_directory -from git.util import finalize_process +from git.util import cwd, finalize_process import os.path as osp from git.compat import is_win -@contextlib.contextmanager -def _chdir(new_dir): - """Context manager to temporarily change directory. Not reentrant.""" - old_dir = os.getcwd() - os.chdir(new_dir) - try: - yield - finally: - os.chdir(old_dir) - - class TestGit(TestBase): @classmethod def setUpClass(cls): @@ -102,7 +90,7 @@ def test_it_executes_git_not_from_cwd(self): print("#!/bin/sh", file=file) os.chmod(impostor_path, 0o755) - with _chdir(tmpdir): + with cwd(tmpdir): self.assertRegex(self.git.execute(["git", "version"]), r"^git version\b") @skipUnless(is_win, "The regression only affected Windows, and this test logic is OS-specific.") From 9da24d46c64eaf4c7db65c0f67324801fafbf30d Mon Sep 17 00:00:00 2001 From: "Wenhan Zhu (Cosmos)" Date: Wed, 6 Sep 2023 20:50:57 -0400 Subject: [PATCH 0556/1790] add test for submodule path not owned by submodule case --- test/test_submodule.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index d906a5d5b..8c98a671e 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -906,6 +906,28 @@ def assert_exists(sm, value=True): assert osp.isdir(sm_module_path) == dry_run # end for each dry-run mode + @with_rw_directory + def test_ignore_non_submodule_file(self, rwdir): + parent = git.Repo.init(rwdir) + + smp = osp.join(rwdir, "module") + os.mkdir(smp) + + with open(osp.join(smp, "a"), "w", encoding="utf-8") as f: + f.write('test\n') + + with open(osp.join(rwdir, ".gitmodules"), "w", encoding="utf-8") as f: + f.write("[submodule \"a\"]\n") + f.write(" path = module\n") + f.write(" url = https://github.com/chaconinc/DbConnector\n") + + parent.git.add(Git.polish_url(osp.join(smp, "a"))) + parent.git.add(Git.polish_url(osp.join(rwdir, ".gitmodules"))) + + parent.git.commit(message='test') + + assert len(parent.submodules) == 0 + @with_rw_directory def test_remove_norefs(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) From fafb4f6651eac242a7e143831fbe23d10beaf89b Mon Sep 17 00:00:00 2001 From: "Wenhan Zhu (Cosmos)" Date: Wed, 6 Sep 2023 20:54:07 -0400 Subject: [PATCH 0557/1790] updated docs to better describe testing procedure with new repo --- AUTHORS | 1 + README.md | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 8ccc09fc0..ba5636db8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -51,4 +51,5 @@ Contributors are: -Luke Twist -Joseph Hale -Santos Gallegos +-Wenhan Zhu Portions derived from other open source works and are clearly marked. diff --git a/README.md b/README.md index 1743bd3d2..94fcc76d9 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,9 @@ See [Issue #525](https://github.com/gitpython-developers/GitPython/issues/525). ### RUNNING TESTS -_Important_: Right after cloning this repository, please be sure to have executed -the `./init-tests-after-clone.sh` script in the repository root. Otherwise -you will encounter test failures. +_Important_: Right after cloning this repository, please be sure to have +executed `git fetch --tags` followed by the `./init-tests-after-clone.sh` +script in the repository root. Otherwise you will encounter test failures. On _Windows_, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` exists in `Git\mingw64\libexec\git-core\`; CYGWIN has no daemon, but should get along fine From c8e303ffd3204195fc7f768f7b17dc5bde3dd53f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Sep 2023 15:35:40 +0200 Subject: [PATCH 0558/1790] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e03213a2b..d87cdbb81 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.34 +3.1.35 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 38e673f3f..6302176bd 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.35 +====== + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/65?closed=1 + 3.1.34 ====== From dba42451187243ffe6d56a08e032dbcbb9ac615f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 16:55:44 -0400 Subject: [PATCH 0559/1790] Fix installation test for Python 3.12 and Windows Starting in Python 3.12, global and virtual Python environments no longer automatically ship setuptools (per the "ensurepip" item in https://docs.python.org/3.12/whatsnew/3.12.html#removed). Projects that use setuptools as a build backend are still supported, including with setup.py using techniques such as "pip install .". In Windows, the "bin" subdir of a virtual environment dir is called "Scripts" instead. Unlike in a global environment (where no names are universal, and "python3" and "pip3" are more common for the Python 3 commands on some popular Unix-like systems), in a virtual environment the "python" and "pip" commands are always present and "python3" and "pip3" are not guaranteed to be present. This commit changes test_installation accordingly. The CI workflows and documentation still need to be updated. --- test/test_installation.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_installation.py b/test/test_installation.py index c092aef5e..fea978ead 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -4,6 +4,7 @@ import ast import os import subprocess +from git.compat import is_win from test.lib import TestBase from test.lib.helper import with_rw_directory @@ -12,8 +13,9 @@ class TestInstallation(TestBase): def setUp_venv(self, rw_dir): self.venv = rw_dir subprocess.run(["virtualenv", self.venv], stdout=subprocess.PIPE) - self.python = os.path.join(self.venv, "bin/python3") - self.pip = os.path.join(self.venv, "bin/pip3") + bin_name = "Scripts" if is_win else "bin" + self.python = os.path.join(self.venv, bin_name, "python") + self.pip = os.path.join(self.venv, bin_name, "pip") self.sources = os.path.join(self.venv, "src") self.cwd = os.path.dirname(os.path.dirname(__file__)) os.symlink(self.cwd, self.sources, target_is_directory=True) @@ -32,14 +34,14 @@ def test_installation(self, rw_dir): msg=result.stderr or result.stdout or "Can't install requirements", ) result = subprocess.run( - [self.python, "setup.py", "install"], + [self.pip, "install", "."], stdout=subprocess.PIPE, cwd=self.sources, ) self.assertEqual( 0, result.returncode, - msg=result.stderr or result.stdout or "Can't build - setup.py failed", + msg=result.stderr or result.stdout or "Can't install project", ) result = subprocess.run([self.python, "-c", "import git"], stdout=subprocess.PIPE, cwd=self.sources) self.assertEqual( From 05229570e8e20a1129464af2f7cb3698f3aa9b48 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 00:12:05 +0300 Subject: [PATCH 0560/1790] Bump GitHub Actions --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9e2d881de..33c6eabe5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,7 +18,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} From 8aa59e702c198e4efd578ca74dda581f73ee258c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 00:13:08 +0300 Subject: [PATCH 0561/1790] Add support for Python 3.12 --- .github/workflows/pythonpackage.yml | 3 ++- setup.py | 2 ++ tox.ini | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 33c6eabe5..73613588d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -25,6 +25,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/setup.py b/setup.py index 1df3b62aa..49d90a02d 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,8 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, diff --git a/tox.ini b/tox.ini index 810baddc3..f7205c5a3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{37, 38, 39, 310, 311} +envlist = flake8, py{37, 38, 39, 310, 311, 312} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From f1b4d14e7d3902135af336ed06a12308dfd40270 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 00:14:52 +0300 Subject: [PATCH 0562/1790] Drop support for EOL Python 3.7 --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73613588d..55c654269 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index e21f53453..ff973bae6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.7+ +* Python 3.8+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 109fec247..8f36481c2 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.7+ +* Python 3.8+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 49d90a02d..b99532814 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.7", + python_requires=">=3.8", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tox.ini b/tox.ini index f7205c5a3..ac3536caa 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{37, 38, 39, 310, 311, 312} +envlist = flake8, py{38, 39, 310, 311, 312} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 72e48aaea59738172ded5c964ddb4f06233ce9b7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 20:11:52 -0400 Subject: [PATCH 0563/1790] Update installation instructions in readme This changes the installation instructions in README.md to recommend "pip install ." instead of "python setup.py install". The former is compatible with Python 3.12 which doesn't have setuptools installed by default (so setup.py, which imports it, can be indirectly but not directly used). This also matches the corresponding change made in the installation unit test. While doing so, I've also clarified the instructions, and added the implied "cd" command as well as the "git fetch --tags" command in the position where a later section was recently updated to mention it should have been run. Using "pip install ." creates the opportunity to pass "-e" to make an editable install, which users who clone the repository to work on changes should do, because the effect of an editable install is only partially simulated by pytest, and so that manual testing of changes actually uses the changes intended for testing. This increases the length and detail of the instructions, so I've added h4 subsections to clarify the separations between them and make it easier for readers to find the part they're looking for. In doing so, I've reordered these subsections accordingly. Because greater detail can create the impression that all important steps are mentioned, I've made the general good advice to use a virtual environment explicit. For brevity, I have not added venv commands. --- README.md | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 94fcc76d9..c44985241 100644 --- a/README.md +++ b/README.md @@ -49,30 +49,49 @@ The installer takes care of installing them for you. ### INSTALL -If you have downloaded the source code: +GitPython and its required package dependencies can be installed in any of the following ways, all of which should typically be done in a [virtual environment](https://docs.python.org/3/tutorial/venv.html). -```bash -python setup.py install -``` +#### From PyPI -or if you want to obtain a copy from the Pypi repository: +To obtain and install a copy [from PyPI](https://pypi.org/project/GitPython/), run: ```bash pip install GitPython ``` -Both commands will install the required package dependencies. +(A distribution package can also be downloaded for manual installation at [the PyPI page](https://pypi.org/project/GitPython/).) + +#### From downloaded source code + +If you have downloaded the source code, run this from inside the unpacked `GitPython` directory: -A distribution package can be obtained for manual installation at: . +```bash +pip install . +``` -If you like to clone from source, you can do it like so: +#### By cloning the source code repository + +To clone the [the GitHub repository](https://github.com/gitpython-developers/GitPython) from source to work on the code, you can do it like so: ```bash git clone https://github.com/gitpython-developers/GitPython -git submodule update --init --recursive +cd GitPython +git fetch --tags ./init-tests-after-clone.sh ``` +If you are cloning [your own fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks), then replace the above `git clone` command with one that gives the URL of your fork. Or use this [`gh`](https://cli.github.com/) command (assuming you have `gh` and your fork is called `GitPython`): + +```bash +gh repo clone GitPython +``` + +Having cloned the repo, create and activate your [virtual environment](https://docs.python.org/3/tutorial/venv.html). Then make an [editable install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): + +```bash +pip install -e . +``` + ### Limitations #### Leakage of System Resources From b095aa9f61691151e68efb972659ac018c8e939d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 20:27:04 -0400 Subject: [PATCH 0564/1790] Use more compatible hashbangs This adds a #! line to the top of setup.py, because it is a script with the executable bit set. Although neither recent nor current documentation in the project recommends to run "./setup.py", this should probably have the intuitive effect of attempting to run the script with a Python interpreter rather than a Unix-style shell. It also uses the "env trick" in init-tests-after-clone.sh so that script runs with whatever bash interpreter is found in a normal PATH search. While "sh" is expected to be found in /bin on all Unix-like systems, that is not always the case for "bash". This change slightly improves compatibility by supporting systems that don't ship with bash but on which it has been installed. --- init-tests-after-clone.sh | 8 +++++--- setup.py | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index e852f3cd9..95ced98b7 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -1,7 +1,9 @@ -#!/bin/bash -e +#!/usr/bin/env bash + +set -e if [[ -z "$TRAVIS" ]]; then - read -p "This operation will destroy locally modified files. Continue ? [N/y]: " answer + read -rp "This operation will destroy locally modified files. Continue ? [N/y]: " answer if [[ ! $answer =~ [yY] ]]; then exit 2 fi @@ -13,4 +15,4 @@ git reset --hard HEAD~1 git reset --hard HEAD~1 git reset --hard HEAD~1 git reset --hard __testing_point__ -git submodule update --init --recursive \ No newline at end of file +git submodule update --init --recursive diff --git a/setup.py b/setup.py index ebece64eb..403482520 100755 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from typing import Sequence from setuptools import setup, find_packages from setuptools.command.build_py import build_py as _build_py From 63c46240334d36129a78866a54694ea8fa5edd16 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 21:51:13 -0400 Subject: [PATCH 0565/1790] Don't duplicate packages across requirements files - Remove "gitdb" from test-requirements.txt, because it already a dependency of the project (listed in requirements.txt, which is used to build the value passed for install_requires in setup.py). - Remove "black" from requirements-dev.txt, because it is listed in test-requirement.txt (which requirements-dev.txt sources). --- requirements-dev.txt | 1 - test-requirements.txt | 2 -- 2 files changed, 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 946b4c94f..f6705341c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,6 @@ # libraries for additional local testing/linting - to be added to test-requirements.txt when all pass flake8-type-checking;python_version>="3.8" # checks for TYPE_CHECKING only imports -black pytest-icdiff # pytest-profiling diff --git a/test-requirements.txt b/test-requirements.txt index 6c6d57060..e202d35c5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,5 +11,3 @@ pytest pytest-cov coverage[toml] pytest-sugar - -gitdb From 3aacb3717ad78ec40e5b168a7ce8109aee6f156e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 23:13:20 -0400 Subject: [PATCH 0566/1790] Use a "test" extra instead of tests_require Because tests_require is deprecated since setuptools 41.5.0 with the intention that it will be removed in some future version (noted in https://setuptools.pypa.io/en/latest/references/keywords.html). It is somewhat unintuitive for GitPython to have a "test" extra, as it makes it so "GitPython[test]" can be specified for installation from PyPI to get test dependencies, even though the PyPI package doesn't include the unit test themselves. However, this makes the statement in README.md that the installer takes care of both requirements.txt and test-requirements.txt dependencies fully true, instead of moving further away from that. Because it is now possible to make an editable GitPython install with test as well as minimal dependencies installed, this commit also updates the readme to document and recommend this. --- README.md | 49 ++++++++++++++++++++++++++++++++++++++++--------- setup.py | 2 +- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c44985241..4ff04c2e4 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,11 @@ gh repo clone GitPython Having cloned the repo, create and activate your [virtual environment](https://docs.python.org/3/tutorial/venv.html). Then make an [editable install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): ```bash -pip install -e . +pip install -e ".[test]" ``` +In the less common case that you do not want to install test dependencies, `pip install -e .` can be used instead. + ### Limitations #### Leakage of System Resources @@ -120,20 +122,49 @@ On _Windows_, make sure you have `git-daemon` in your PATH. For MINGW-git, the ` exists in `Git\mingw64\libexec\git-core\`; CYGWIN has no daemon, but should get along fine with MINGW's. -Ensure testing libraries are installed. -In the root directory, run: `pip install -r test-requirements.txt` +#### Install test dependencies + +Ensure testing libraries are installed. This is taken care of already if you installed with: + +```bash +pip install -e ".[test]" +``` + +Otherwise, you can run: + +```bash +pip install -r test-requirements.txt +``` + +#### Test commands + +To test, run: -To lint, run: `pre-commit run --all-files` +```bash +pytest +``` + +To lint, run: -To typecheck, run: `mypy -p git` +```bash +pre-commit run --all-files +``` -To test, run: `pytest` +To typecheck, run: -For automatic code formatting run: `black git` +```bash +mypy -p git +``` + +For automatic code formatting, run: + +```bash +black git +``` -Configuration for flake8 is in the ./.flake8 file. +Configuration for flake8 is in the `./.flake8` file. -Configurations for mypy, pytest and coverage.py are in ./pyproject.toml. +Configurations for `mypy`, `pytest`, `coverage.py`, and `black` are in `./pyproject.toml`. The same linting and testing will also be performed against different supported python versions upon submitting a pull request (or on each push if you have a fork with a "main" branch and actions enabled). diff --git a/setup.py b/setup.py index 403482520..5516a67d1 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,7 @@ def build_py_modules(basedir: str, excludes: Sequence = ()) -> Sequence: package_dir={"git": "git"}, python_requires=">=3.7", install_requires=requirements, - tests_require=requirements + test_requirements, + extras_require={"test": test_requirements}, zip_safe=False, long_description=long_description, long_description_content_type="text/markdown", From e1d8b40e91eedabc3a2c2c1ac653f86b662323a9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 23:33:06 -0400 Subject: [PATCH 0567/1790] Use "build" for building Instead of directly running setup.py. This allows Python 3.12 (as well as previous versions) to be used for building. Although setuptools could be added as a development dependency to run setup.py, using "build" instead is recommended in https://setuptools.pypa.io/en/latest/userguide/quickstart.html. Those docs likewise recommend only listing "wheel" in the build-system section of pyproject.toml if setup.py actually imports the wheel module. So this removes that. (Running "make release", which now uses "build", will continue to build wheels.) The "build" package is not conceptually a testing dependency, but test-requirements.txt is currently the de facto list of all stable development dependencies for regular use. --- Makefile | 2 +- pyproject.toml | 2 +- test-requirements.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2af8de084..6bde85af1 100644 --- a/Makefile +++ b/Makefile @@ -16,5 +16,5 @@ release: clean force_release: clean git push --tags origin main - python3 setup.py sdist bdist_wheel + python -m build --sdist --wheel twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml index 32c9d4a26..42bb31eda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "wheel"] +requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] diff --git a/test-requirements.txt b/test-requirements.txt index e202d35c5..67496031e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,6 +5,7 @@ black pre-commit +build virtualenv pytest From b9b6d8c07eb9afb88f092e507db9ae467e5e0986 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Sep 2023 23:55:26 -0400 Subject: [PATCH 0568/1790] Ungroup and sort test_requirements.txt This is cleanup related to the previous commit. As that file grows, it is harder to tell immediately if a particular package is in it when not alphabetized. (The groups were also not intuitive, with ddt listed separately from other unit test related dependencies.) --- test-requirements.txt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 67496031e..214ad78ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,14 +1,10 @@ +black +build +coverage[toml] ddt>=1.1.1, !=1.4.3 mypy - -black - pre-commit - -build -virtualenv - pytest pytest-cov -coverage[toml] pytest-sugar +virtualenv From 21c5f8749590480de7cbcfe87352b60759328a75 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 00:08:58 -0400 Subject: [PATCH 0569/1790] Don't preinstall dependencies in test_installation This removes the step in test_installation that did the equivalent of "pip install -r requirements.txt", because installing GitPython is sufficient to install all its required dependencies, and it is more important to test that than to test requirements.txt directly. Removing this causes the test to fail if installing the project doesn't entail installation of the requirements necessary to import the git module or to cause gitdb to be found in a sys.path search. --- test/test_installation.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/test_installation.py b/test/test_installation.py index fea978ead..d856ebc94 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -23,16 +23,6 @@ def setUp_venv(self, rw_dir): @with_rw_directory def test_installation(self, rw_dir): self.setUp_venv(rw_dir) - result = subprocess.run( - [self.pip, "install", "-r", "requirements.txt"], - stdout=subprocess.PIPE, - cwd=self.sources, - ) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Can't install requirements", - ) result = subprocess.run( [self.pip, "install", "."], stdout=subprocess.PIPE, From 6b54890cfc14f1c574e8bf85d1ebf9be4c92be3e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 01:18:44 -0400 Subject: [PATCH 0570/1790] Test changed setup, and Python 3.12, on CI Key changes: - Update the two CI workflows to install the project and its dependencies in accordance with the changed recommendations in README.md. (This is to test that those recommendations work, which the changed test_installation test case partially but not completely tests. The old approach to installation still works too, so this change on CI is not required to keep CI working.) - Add Python 3.12 to the CI test matrix in pythonpackage.yml, testing it on Ubuntu. (The Cygwin workflow still tests only 3.9.) Maintenance changes, made to avoid decreasing readability with the other changes (and hopefully even increase it somewhat): - Separate commands into more steps, grouping them by more specific purposes. - Decrease the ways the two workflows differ from each other that do not represent actual intended behavioral differences. This is to make the important differences easier to stop, and to make it easier to determine when the same change has or has not been made to both workflows. --- .github/workflows/cygwin-test.yml | 41 ++++++++++++++++++------ .github/workflows/pythonpackage.yml | 48 +++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 808dc5608..ceb988cec 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -12,38 +12,61 @@ jobs: SHELLOPTS: igncr TMP: "/tmp" TEMP: "/tmp" + defaults: + run: + shell: bash.exe -eo pipefail -o igncr "{0}" steps: - name: Force LF line endings run: git config --global core.autocrlf input + - uses: actions/checkout@v4 with: fetch-depth: 9999 + - uses: cygwin/cygwin-install-action@v4 with: packages: python39 python39-pip python39-virtualenv git + + - name: Show python and git versions + run: | + set -x + /usr/bin/python --version + /usr/bin/git version + - name: Tell git to trust this repo - shell: bash.exe -eo pipefail -o igncr "{0}" run: | /usr/bin/git config --global --add safe.directory "$(pwd)" - - name: Install dependencies and prepare tests - shell: bash.exe -eo pipefail -o igncr "{0}" + + - name: Prepare this repo for tests run: | set -x - /usr/bin/python -m pip install --upgrade pip setuptools wheel - /usr/bin/python --version; /usr/bin/git --version + /usr/bin/git submodule update --init --recursive /usr/bin/git fetch --tags - /usr/bin/python -m pip install -r requirements.txt - /usr/bin/python -m pip install -r test-requirements.txt TRAVIS=yes ./init-tests-after-clone.sh + + - name: Further prepare git configuration for tests + run: | + set -x + /usr/bin/git config --global user.email "travis@ci.com" /usr/bin/git config --global user.name "Travis Runner" # If we rewrite the user's config by accident, we will mess it up # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig + + - name: Update PyPA packages + run: | + set -x + /usr/bin/python -m pip install --upgrade pip setuptools wheel + + - name: Install project and test dependencies + run: | + set -x + /usr/bin/python -m pip install ".[test]" + - name: Test with pytest - shell: bash.exe -eo pipefail -o igncr "{0}" run: | + set -x /usr/bin/python -m pytest - continue-on-error: false diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a6af507d1..9f15ceb2e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,42 +15,70 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - experimental: false + - python-version: "3.12" + experimental: true steps: - uses: actions/checkout@v4 with: fetch-depth: 9999 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies and prepare tests + allow-prereleases: ${{ matrix.experimental }} + + - name: Show python and git versions + run: | + set -x + python --version + git version + + - name: Prepare this repo for tests run: | set -x - python -m pip install --upgrade pip setuptools wheel - python --version; git --version git submodule update --init --recursive git fetch --tags --force - - pip install -r requirements.txt - pip install -r test-requirements.txt TRAVIS=yes ./init-tests-after-clone.sh + - name: Prepare git configuration for tests + run: | + set -x + git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" # If we rewrite the user's config by accident, we will mess it up # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig + - name: Update PyPA packages + run: | + set -x + + python -m pip install --upgrade pip + if pip freeze --all | grep --quiet '^setuptools=='; then + # Python prior to 3.12 ships setuptools. Upgrade it if present. + python -m pip install --upgrade setuptools + fi + python -m pip install --upgrade wheel + + - name: Install project and test dependencies + run: | + set -x + pip install ".[test]" + - name: Check types with mypy - # With new versions of pypi new issues might arise. This is a problem if there is nobody able to fix them, - # so we have to ignore errors until that changes. - continue-on-error: true run: | set -x mypy -p git + # With new versions of mypy new issues might arise. This is a problem if there is nobody able to fix them, + # so we have to ignore errors until that changes. + continue-on-error: true - name: Test with pytest run: | From 055355d0028147c27b0423d713803748e60af6c3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 04:23:25 -0400 Subject: [PATCH 0571/1790] Don't use "set -x" for "pytest" command on Cygwin The omission of "set -x" was intentional and is currently necessary on Cygwin (but not on Ubuntu), per aafb92a. --- .github/workflows/cygwin-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index ceb988cec..2e2e6d12d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -68,5 +68,4 @@ jobs: - name: Test with pytest run: | - set -x /usr/bin/python -m pytest From 32d12aa03aff193603aad1d6f08669a70607beb9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 18:38:38 +0300 Subject: [PATCH 0572/1790] Bump GitHub Actions --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0d039adae..1cefabc3e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,7 +18,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} From d7fc1fd3d154c809b1c021ae3fa564e9fe4b4859 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 18:38:56 +0300 Subject: [PATCH 0573/1790] Add support for Python 3.12 --- .github/workflows/pythonpackage.yml | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 1cefabc3e..dab41d093 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -25,6 +25,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/setup.py b/setup.py index 521484978..61b572768 100755 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ] ) From 875acb45cd8e9353c1911717bd2cd05b3e40ed05 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 18:39:33 +0300 Subject: [PATCH 0574/1790] Drop support for EOL Python 3.7 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index dab41d093..98e670c22 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/setup.py b/setup.py index 61b572768..57341223b 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.7', + python_requires='>=3.8', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,7 +35,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From a352404576552116bd16d9ca40cbcb903d3af020 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 13:24:40 -0400 Subject: [PATCH 0575/1790] List Python 3.12 as supported in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5516a67d1..44623d049 100755 --- a/setup.py +++ b/setup.py @@ -124,5 +124,6 @@ def build_py_modules(basedir: str, excludes: Sequence = ()) -> Sequence: "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], ) From 415a8ebdf8d0648cd17218e817963721d8df54da Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 13:28:16 -0400 Subject: [PATCH 0576/1790] Small clarity improvements in setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 44623d049..bc53bf6c8 100755 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ import os import sys -with open(os.path.join(os.path.dirname(__file__), "VERSION")) as v: - VERSION = v.readline().strip() +with open(os.path.join(os.path.dirname(__file__), "VERSION")) as ver_file: + VERSION = ver_file.readline().strip() with open("requirements.txt") as reqs_file: requirements = reqs_file.read().splitlines() @@ -49,7 +49,7 @@ def _stamp_version(filename: str) -> None: with open(filename) as f: for line in f: if "__version__ =" in line: - line = line.replace("\"git\"", "'%s'" % VERSION) + line = line.replace('"git"', "'%s'" % VERSION) found = True out.append(line) except OSError: From 8ac188497e7ac1ef2e89c984ac3728319b625e1a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 18:14:06 -0400 Subject: [PATCH 0577/1790] Enable Dependabot version updates for Actions This enables Dependabot version updates for GitHub Actions only (not Python dependencies), using the exact same configuration as in GitPython. --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..203f3c889 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 4eef3ecd5b2eb65f1e98457a35df8700166f67b0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 00:41:04 -0400 Subject: [PATCH 0578/1790] Have actions/checkout do the full fetch Setting the "fetch-depth" to 0 does a deep (i.e., ordinary) fetch, fetching all commits and tags. Setting "submodules" to "recursive" clones and checks out all submodules. These options allow commands that were doing those things to be removed from the later steps. --- .github/workflows/cygwin-test.yml | 6 ++---- .github/workflows/pythonpackage.yml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 2e2e6d12d..533022b91 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -22,7 +22,8 @@ jobs: - uses: actions/checkout@v4 with: - fetch-depth: 9999 + fetch-depth: 0 + submodules: recursive - uses: cygwin/cygwin-install-action@v4 with: @@ -41,9 +42,6 @@ jobs: - name: Prepare this repo for tests run: | set -x - - /usr/bin/git submodule update --init --recursive - /usr/bin/git fetch --tags TRAVIS=yes ./init-tests-after-clone.sh - name: Further prepare git configuration for tests diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9f15ceb2e..cce39d17a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,7 +24,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 9999 + fetch-depth: 0 + submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -41,9 +42,6 @@ jobs: - name: Prepare this repo for tests run: | set -x - - git submodule update --init --recursive - git fetch --tags --force TRAVIS=yes ./init-tests-after-clone.sh - name: Prepare git configuration for tests From 5f128e8c23e730748eb73bc2eb9f8ee552064e93 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 01:42:40 -0400 Subject: [PATCH 0579/1790] Move effect of "set -x" into default shell command This also adds "--noprofile --norc" to the Cygwin shell command as a speed optimization (bash doesn't need to source its scripts). That only changes the Cygwin workflow; in the Ubuntu workflow, "--noprofile --norc" had already been included by default when no shell was specified, so having it there is to *keep* the optimized behavior that was already in use. --- .github/workflows/cygwin-test.yml | 11 +++-------- .github/workflows/pythonpackage.yml | 13 +++---------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 533022b91..962791ae7 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -14,7 +14,7 @@ jobs: TEMP: "/tmp" defaults: run: - shell: bash.exe -eo pipefail -o igncr "{0}" + shell: bash.exe --noprofile --norc -exo pipefail -o igncr "{0}" steps: - name: Force LF line endings @@ -31,23 +31,19 @@ jobs: - name: Show python and git versions run: | - set -x /usr/bin/python --version /usr/bin/git version - name: Tell git to trust this repo run: | - /usr/bin/git config --global --add safe.directory "$(pwd)" + /usr/bin/git config --global --add safe.directory "$(pwd)" - name: Prepare this repo for tests run: | - set -x TRAVIS=yes ./init-tests-after-clone.sh - name: Further prepare git configuration for tests run: | - set -x - /usr/bin/git config --global user.email "travis@ci.com" /usr/bin/git config --global user.name "Travis Runner" # If we rewrite the user's config by accident, we will mess it up @@ -56,14 +52,13 @@ jobs: - name: Update PyPA packages run: | - set -x /usr/bin/python -m pip install --upgrade pip setuptools wheel - name: Install project and test dependencies run: | - set -x /usr/bin/python -m pip install ".[test]" - name: Test with pytest run: | + set +x /usr/bin/python -m pytest diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cce39d17a..a5467ef94 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,6 +20,9 @@ jobs: - experimental: false - python-version: "3.12" experimental: true + defaults: + run: + shell: /bin/bash --noprofile --norc -exo pipefail {0} steps: - uses: actions/checkout@v4 @@ -35,19 +38,15 @@ jobs: - name: Show python and git versions run: | - set -x python --version git version - name: Prepare this repo for tests run: | - set -x TRAVIS=yes ./init-tests-after-clone.sh - name: Prepare git configuration for tests run: | - set -x - git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" # If we rewrite the user's config by accident, we will mess it up @@ -56,8 +55,6 @@ jobs: - name: Update PyPA packages run: | - set -x - python -m pip install --upgrade pip if pip freeze --all | grep --quiet '^setuptools=='; then # Python prior to 3.12 ships setuptools. Upgrade it if present. @@ -67,12 +64,10 @@ jobs: - name: Install project and test dependencies run: | - set -x pip install ".[test]" - name: Check types with mypy run: | - set -x mypy -p git # With new versions of mypy new issues might arise. This is a problem if there is nobody able to fix them, # so we have to ignore errors until that changes. @@ -80,12 +75,10 @@ jobs: - name: Test with pytest run: | - set -x pytest continue-on-error: false - name: Documentation run: | - set -x pip install -r doc/requirements.txt make -C doc html From e7937aefaf49c72b5f4432cc1f303ed23889589a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 02:31:34 -0400 Subject: [PATCH 0580/1790] Test installing project on CI This changes the CI workflow to install the project itself to get its dependency, rather than installing the dependency from requirements.txt. This is the more important thing to test, because it verifies that the project is installable and effectively declares the dependencies it needs. --- .github/workflows/pythonpackage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 98e670c22..efa04a901 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -26,10 +26,10 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: true - - name: Install dependencies + - name: Install project and dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install . - name: Lint with flake8 run: | pip install flake8 From 60f7f94607996e05292896e8105ef3780779959d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 03:38:59 -0400 Subject: [PATCH 0581/1790] Fix mkdir race condition in LooseObjectDB.store This replaces the conditional call to os.mkdir that raises an unintended FileExistsError if the directory is created between the check and the os.mkdir call, using a single os.makedirs call instead, with exist_ok=True. This way, we attempt creation in a way that produces no error if the directory is already present, while still raising FileExistsError if a non-directory filesystem entry (such as a regular file) is present where we want the directory to be. --- gitdb/db/loose.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4ef775049..7ea6fef3d 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -8,7 +8,6 @@ ObjectDBW ) - from gitdb.exc import ( BadObject, AmbiguousObjectName @@ -33,10 +32,8 @@ bin_to_hex, exists, chmod, - isdir, isfile, remove, - mkdir, rename, dirname, basename, @@ -222,8 +219,7 @@ def store(self, istream): if tmp_path: obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) + os.makedirs(obj_dir, exist_ok=True) # END handle destination directory # rename onto existing doesn't work on NTFS if isfile(obj_path): From 9d126ce78e69fb8aba28ae89adde69298a3066a4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 04:33:18 -0400 Subject: [PATCH 0582/1790] Don't cancel other jobs from the 3.12 job failing Because 3.12 is still a release candidate and if tests fail for it then one would always want to know if/how other versions also fail. This also allows actions/setup-python to install a prerelease for 3.12 only, not for other releases. --- .github/workflows/pythonpackage.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index efa04a901..c59e618ac 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -16,6 +16,11 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - experimental: false + - python-version: "3.12" + experimental: true + continue-on-error: ${{ matrix.experimental }} steps: - uses: actions/checkout@v4 @@ -25,7 +30,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - allow-prereleases: true + allow-prereleases: ${{ matrix.experimental }} - name: Install project and dependencies run: | python -m pip install --upgrade pip From 919d3cce57d0a3c398dc4ddbacbbd23942a6ff4c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 04:17:52 -0400 Subject: [PATCH 0583/1790] Use actions/checkout feature to fetch all commits Setting the "fetch-depth" to a positive value fetches that many commits back (and the default value is 1), but setting it to 0 fetches all commits, as in a (deep) normal fetch. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index efa04a901..14c7ecbb4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1000 + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From bd21ed46addf55fea5059d44e0477a785f4a664f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 05:18:27 -0400 Subject: [PATCH 0584/1790] Revert "Drop support for EOL Python 3.7" This brings back Python 3.7 support (allowing it to be installed on 3.7, and testing on 3.7 on CI), even though 3.7 is end-of-life, because support for 3.7 is not being dropped by GitPython yet, and there is value in keeping the version ranges supported by GitPython and gitdb consistent. This reverts commit 875acb45cd8e9353c1911717bd2cd05b3e40ed05. --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3d06b4aec..6849763e1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - python-version: "3.12" diff --git a/setup.py b/setup.py index 57341223b..61b572768 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.8', + python_requires='>=3.7', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,6 +35,7 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From d99b2d4629f2257ac719f77b93e1a2d7affd08c6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Sep 2023 11:26:57 +0200 Subject: [PATCH 0585/1790] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d87cdbb81..b402c1a8b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.35 +3.1.36 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6302176bd..06ec4b72c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ========= +3.1.36 +====== + +Note that this release should be a no-op, it's mainly for testing the changed release-process. + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/66?closed=1 + 3.1.35 ====== From f86f09e12c5616680e19b45c940144520349c9e3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Sep 2023 12:53:21 +0200 Subject: [PATCH 0586/1790] Make publish process possible on MacOS There the system python interpreter is referred to as `python3`, at least when installed by homebrew. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6bde85af1..c3c00e948 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,6 @@ release: clean make force_release force_release: clean - git push --tags origin main - python -m build --sdist --wheel + python3 -m build --sdist --wheel twine upload dist/* + git push --tags origin main From 5343aa01e9d90481e4570797e99faf6a98ba8f6c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 13:39:34 -0400 Subject: [PATCH 0587/1790] Let "make" install build and twine if in a virtual environment If a virtual environment (created by venv or virtualenv) is active, running "make release" or "make force_release" now automatically installs/upgrades the "build" and "twine" packages in it. This is only done if "make" is run in a virtual environment. This can be a fresh environment: neither the project nor its dependencies need to be installed in it. Because the "build" module is not currently used in any tests and running "make" in a virtual environment takes care of installing "build" (and "twine"), "build" is now removed from test-requirements.txt. The publishing instructions in the readme are updated accordingly, to mention the optional step of creating and activating a virtual environment, and to briefly clarify why one might want to do that. Running "make" outside a virtual environment remains supported, except that, due to recent changes, whatever environment it is run in needs to have a usable "build" module. --- Makefile | 2 ++ README.md | 14 ++++++++------ test-requirements.txt | 1 - 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index c3c00e948..f2cbf826a 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ release: clean make force_release force_release: clean + # IF we're in a virtual environment, add build tools + test -z "$$VIRTUAL_ENV" || pip install -U build twine python3 -m build --sdist --wheel twine upload dist/* git push --tags origin main diff --git a/README.md b/README.md index 4ff04c2e4..ca470a851 100644 --- a/README.md +++ b/README.md @@ -188,13 +188,15 @@ Please have a look at the [contributions file][contributing]. ### How to make a new release -- Update/verify the **version** in the `VERSION` file -- Update/verify that the `doc/source/changes.rst` changelog file was updated -- Commit everything -- Run `git tag -s ` to tag the version in Git -- Run `make release` +- Update/verify the **version** in the `VERSION` file. +- Update/verify that the `doc/source/changes.rst` changelog file was updated. +- Commit everything. +- Run `git tag -s ` to tag the version in Git. +- _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) using `venv` or `virtualenv`.\ +(When run in a virtual environment, the next step will automatically take care of installing `build` and `twine` in it.) +- Run `make release`. - Close the milestone mentioned in the _changelog_ and create a new one. _Do not reuse milestones by renaming them_. -- Got to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. +- Go to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. ### How to verify a release (DEPRECATED) diff --git a/test-requirements.txt b/test-requirements.txt index 214ad78ec..62f409824 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,4 @@ black -build coverage[toml] ddt>=1.1.1, !=1.4.3 mypy From 74d9c08bc4b8a61cd9f31e2294e1ff74d7b2be08 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 23:11:43 -0400 Subject: [PATCH 0588/1790] Better explain the env_case test This expands and adds comments in test_it_avoids_upcasing_unrelated_environment_variable_names and its supporting fixture env_case.py so it is clear exactly what is being tested and how/why the test works to test it. --- test/fixtures/env_case.py | 7 ++++++- test/test_git.py | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/test/fixtures/env_case.py b/test/fixtures/env_case.py index 120e59289..fe85ac41d 100644 --- a/test/fixtures/env_case.py +++ b/test/fixtures/env_case.py @@ -1,13 +1,18 @@ +# Steps 3 and 4 for test_it_avoids_upcasing_unrelated_environment_variable_names. + import subprocess import sys +# Step 3a: Import the module, in case that upcases the environment variable name. import git _, working_dir, env_var_name = sys.argv -# Importing git should be enough, but this really makes sure Git.execute is called. +# Step 3b: Use Git.execute explicitly, in case that upcases the environment variable. +# (Importing git should be enough, but this ensures Git.execute is called.) repo = git.Repo(working_dir) # Hold the reference. git.Git(repo.working_dir).execute(["git", "version"]) +# Step 4: Create the non-Python grandchild that accesses the variable case-sensitively. print(subprocess.check_output(["set", env_var_name], shell=True, text=True)) diff --git a/test/test_git.py b/test/test_git.py index f1d35a355..3b4a633b6 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -98,15 +98,25 @@ def test_it_avoids_upcasing_unrelated_environment_variable_names(self): old_name = "28f425ca_d5d8_4257_b013_8d63166c8158" if old_name == old_name.upper(): raise RuntimeError("test bug or strange locale: old_name invariant under upcasing") - os.putenv(old_name, "1") # It has to be done this lower-level way to set it lower-case. + # Step 1: Set the environment variable in this parent process. Because os.putenv is a thin + # wrapper around a system API, os.environ never sees the variable in this parent + # process, so the name is not upcased even on Windows. + os.putenv(old_name, "1") + + # Step 2: Create the child process that inherits the environment variable. It will see it + # in os.environ with an upcased name, but if it is not mutated through os.environ + # then it will pass it on to its own child processes with the original name. The + # child process will use GitPython, and we are testing that it passes the variable + # with the exact original name to its own child processes. cmdline = [ sys.executable, fixture_path("env_case.py"), self.rorepo.working_dir, old_name, ] - pair_text = subprocess.check_output(cmdline, shell=False, text=True) + pair_text = subprocess.check_output(cmdline, shell=False, text=True) # Steps 3 and 4. + new_name = pair_text.split("=")[0] self.assertEqual(new_name, old_name) From a59aaead478213b6fd83dd39c607baa2ead97e09 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 23:14:51 -0400 Subject: [PATCH 0589/1790] Condense an overly long comment To better focus on the key information. --- test/test_git.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 3b4a633b6..60cd60bd9 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -104,11 +104,9 @@ def test_it_avoids_upcasing_unrelated_environment_variable_names(self): # process, so the name is not upcased even on Windows. os.putenv(old_name, "1") - # Step 2: Create the child process that inherits the environment variable. It will see it - # in os.environ with an upcased name, but if it is not mutated through os.environ - # then it will pass it on to its own child processes with the original name. The - # child process will use GitPython, and we are testing that it passes the variable - # with the exact original name to its own child processes. + # Step 2: Create the child process that inherits the environment variable. The child uses + # GitPython, and we are testing that it passes the variable with the exact original + # name to its own child process (the grandchild). cmdline = [ sys.executable, fixture_path("env_case.py"), From 22b8dba6b4c73d04f3dbfabbb858fb5316fec711 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 23:28:14 -0400 Subject: [PATCH 0590/1790] Improve git.util.cwd docstring This frames the reentrancy claim in terms of what is similar to and different from contextlib.chdir (which we would just use, but it is only available on Python 3.11 and later). --- git/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index dee467dd3..636e79806 100644 --- a/git/util.py +++ b/git/util.py @@ -150,7 +150,10 @@ def wrapper(self: "Remote", *args: Any, **kwargs: Any) -> T: @contextlib.contextmanager def cwd(new_dir: PathLike) -> Generator[PathLike, None, None]: - """Context manager to temporarily change directory. Not reentrant.""" + """Context manager to temporarily change directory. + + This is similar to contextlib.chdir introduced in Python 3.11, but the context + manager object returned by a single call to this function is not reentrant.""" old_dir = os.getcwd() os.chdir(new_dir) try: From c547555026e099defd361e95e8c06acd5f2c0a2f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Sep 2023 00:56:01 -0400 Subject: [PATCH 0591/1790] Clarify test relationship to env_case.py fixture --- test/test_git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 60cd60bd9..4d57a2d86 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -109,11 +109,11 @@ def test_it_avoids_upcasing_unrelated_environment_variable_names(self): # name to its own child process (the grandchild). cmdline = [ sys.executable, - fixture_path("env_case.py"), + fixture_path("env_case.py"), # Contains steps 3 and 4. self.rorepo.working_dir, old_name, ] - pair_text = subprocess.check_output(cmdline, shell=False, text=True) # Steps 3 and 4. + pair_text = subprocess.check_output(cmdline, shell=False, text=True) # Run steps 3 and 4. new_name = pair_text.split("=")[0] self.assertEqual(new_name, old_name) From 4aafbbf4a21d3a46bed6e2dafb46354f7ce6e346 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Sep 2023 00:58:45 -0400 Subject: [PATCH 0592/1790] Remove spurious executable permissions This unsets the executable bit on test/fixtures/diff_mode_only, which was the only of the fixture files to be marked executable, and which is not a script. --- test/fixtures/diff_mode_only | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 test/fixtures/diff_mode_only diff --git a/test/fixtures/diff_mode_only b/test/fixtures/diff_mode_only old mode 100755 new mode 100644 From d6d8ecdb10d5c9d1820c4b314694b54f7755757f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Sep 2023 08:02:41 +0200 Subject: [PATCH 0593/1790] leave another note in Makefile to help remember using virtualenv --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f2cbf826a..389337d08 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,6 @@ release: clean force_release: clean # IF we're in a virtual environment, add build tools test -z "$$VIRTUAL_ENV" || pip install -U build twine - python3 -m build --sdist --wheel + python3 -m build --sdist --wheel || echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" twine upload dist/* git push --tags origin main From c86284565ce97de07870379e52a567343d7c4662 Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:12:58 -0600 Subject: [PATCH 0594/1790] Fix dynamically-set __all__ variable --- git/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 6196a42d7..cc0ca2136 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -61,8 +61,19 @@ def _init_externals() -> None: # } END imports -__all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] - +# __all__ must be statically defined by py.typed support +# __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] +__all__ = ['BadName', 'safe_decode', + 'remove_password_if_present', 'List', 'Sequence', 'Tuple', 'Union', 'TYPE_CHECKING', + 'PathLike', 'GitError', 'InvalidGitRepositoryError', 'WorkTreeRepositoryUnsupported', + 'NoSuchPathError', 'UnsafeProtocolError', 'UnsafeOptionError', 'CommandError', 'GitCommandNotFound', + 'GitCommandError', 'CheckoutError', 'CacheError', 'UnmergedEntriesError', 'HookExecutionError', + 'RepositoryDirtyError', 'Optional', 'GitConfigParser', 'Object', 'IndexObject', 'Blob', 'Commit', + 'Submodule', 'UpdateProgress', 'RootModule', 'RootUpdateProgress', 'TagObject', 'TreeModifier', + 'Tree', 'SymbolicReference', 'Reference', 'HEAD', 'Head', 'TagReference', 'Tag', 'RemoteReference', + 'RefLog', 'RefLogEntry', 'Diffable', 'DiffIndex', 'Diff', 'NULL_TREE', 'GitCmdObjectDB', 'GitDB', + 'Git', 'Repo', 'RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote', 'IndexFile', 'StageType', + 'BlobFilter', 'BaseIndexEntry', 'IndexEntry', 'LockFile', 'BlockingLockFile', 'Stats', 'Actor', 'rmtree'] # { Initialize git executable path GIT_OK = None From 6fb231828ea845e7f0b80e52280f3efb98a25545 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 13:55:56 -0400 Subject: [PATCH 0595/1790] Reference HEAD in Makefile (more portable than head) This fixes "fatal: ambiguous argument 'head'", which occurs on some systems, inclding GNU/Linux systems, with "git rev-parse head". --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 389337d08..f3a5d3749 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,9 @@ clean: release: clean # Check if latest tag is the current head we're releasing echo "Latest tag = $$(git tag | sort -nr | head -n1)" - echo "HEAD SHA = $$(git rev-parse head)" + echo "HEAD SHA = $$(git rev-parse HEAD)" echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse HEAD)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" make force_release force_release: clean From 335d03b174569b4bc840e2020886a5f65adc56b7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 14:54:04 -0400 Subject: [PATCH 0596/1790] Have Makefile use git tag to sort the tags --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f3a5d3749..592f5658d 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,10 @@ clean: release: clean # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag | sort -nr | head -n1)" + echo "Latest tag = $$(git tag -l '[0-9]*' --sort=-v:refname | head -n1)" echo "HEAD SHA = $$(git rev-parse HEAD)" - echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse HEAD)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + echo "Latest tag SHA = $$(git tag -l '[0-9]*' --sort=-v:refname | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse HEAD)" = "$$(git tag -l '[0-9]*' --sort=-v:refname | head -n1 | xargs git rev-parse)" make force_release force_release: clean From b1c61d933d36a8e7fb6fb4109d2b07bd06bfbf32 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 05:44:16 -0400 Subject: [PATCH 0597/1790] Make "git tag" sort our SemVer-ish tags correctly This sorts numerically for each of major, minor, and patch, rather than, e.g., rating 2.1.15 as a higher version than 2.1.2. It also rates things like X-beta and X-rc as lower versions than X, but X-patched (not SemVer, but present in this project) as higher versions than X. --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 592f5658d..4e1927a9c 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,14 @@ clean: release: clean # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag -l '[0-9]*' --sort=-v:refname | head -n1)" - echo "HEAD SHA = $$(git rev-parse HEAD)" - echo "Latest tag SHA = $$(git tag -l '[0-9]*' --sort=-v:refname | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse HEAD)" = "$$(git tag -l '[0-9]*' --sort=-v:refname | head -n1 | xargs git rev-parse)" + @config_opts="$$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" && \ + latest_tag=$$(git $$config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1) && \ + head_sha=$$(git rev-parse HEAD) latest_tag_sha=$$(git rev-parse "$$latest_tag") && \ + printf '%-14s = %s\n' 'Latest tag' "$$latest_tag" \ + 'HEAD SHA' "$$head_sha" \ + 'Latest tag SHA' "$$latest_tag_sha" && \ + test "$$head_sha" = "$$latest_tag_sha" + make force_release force_release: clean From cc202cc0fcbbb365e86a8dbc99bb5d6381619671 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 06:13:44 -0400 Subject: [PATCH 0598/1790] Improve when and how Makefile suggests virtual env The avoids showing the message when the build command was already run in a virtual environment. It also keeps the command failing, so the subsequent twine command is not attempted. (Just adding "|| echo ..." caused the command to succeed, because "echo ..." itself succeeds except in the rare case it cannot write to standard output.) --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4e1927a9c..1c2c03a88 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,13 @@ release: clean force_release: clean # IF we're in a virtual environment, add build tools test -z "$$VIRTUAL_ENV" || pip install -U build twine - python3 -m build --sdist --wheel || echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" + + # Build the sdist and wheel that will be uploaded to PyPI. + python3 -m build --sdist --wheel || \ + test -z "$$VIRTUAL_ENV" && \ + echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" && \ + false + + # Upload to PyPI and push the tag. twine upload dist/* git push --tags origin main From b54c34647bf6281d7f5f757d9850484bdb25f800 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 06:33:10 -0400 Subject: [PATCH 0599/1790] Use "python" in the virtual env, "python3" outside This changes the build command to run with "python" when in a virtual environment, since all virtual environments support this even when "python" outside it is absent or refers to the wrong version. On Windows, virtual environments don't contain a python3 command, but a global python3 command may be present, so the errors are confusing. This fixes that by avoiding such errors altogether. --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1c2c03a88..cd5eba36b 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,13 @@ force_release: clean test -z "$$VIRTUAL_ENV" || pip install -U build twine # Build the sdist and wheel that will be uploaded to PyPI. - python3 -m build --sdist --wheel || \ - test -z "$$VIRTUAL_ENV" && \ + if test -n "$$VIRTUAL_ENV"; then \ + python -m build --sdist --wheel; \ + else \ + python3 -m build --sdist --wheel || \ echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" && \ - false + false; \ + fi # Upload to PyPI and push the tag. twine upload dist/* From ae9405a339e7b01fb539e1837e4c3f450250cfb7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 06:41:03 -0400 Subject: [PATCH 0600/1790] LF line endings for scripts that may need them This fixes how init-tests-after-clone.sh appears in .gitattributes so it gets LF (Unix-style) line endings on all systems as intended, and adds Makefile to be treated the same way. --- .gitattributes | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 6d2618f2f..739b2be29 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ test/fixtures/* eol=lf -init-tests-after-clone.sh +init-tests-after-clone.sh eol=lf +Makefile eol=lf From f5da163bed2ababec006c0806187070341cc390e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 07:45:26 -0400 Subject: [PATCH 0601/1790] Have "make release" check other release preconditions As documented in the release instructions in README.md. --- Makefile | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cd5eba36b..52d080788 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,29 @@ clean: rm -rf build/ dist/ .eggs/ .tox/ release: clean - # Check if latest tag is the current head we're releasing - @config_opts="$$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" && \ + # Check that VERSION and changes.rst exist and have no uncommitted changes + test -f VERSION + test -f doc/source/changes.rst + git status -s VERSION doc/source/changes.rst + @test -z "$$(git status -s VERSION doc/source/changes.rst)" + + # Check that ALL changes are commited (can comment out if absolutely necessary) + git status -s + @test -z "$$(git status -s)" + + # Check that latest tag matches version and is the current head we're releasing + @version_file="$$(cat VERSION)" && \ + changes_file="$$(awk '/^[0-9]/ {print $$0; exit}' doc/source/changes.rst)" && \ + config_opts="$$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" && \ latest_tag=$$(git $$config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1) && \ head_sha=$$(git rev-parse HEAD) latest_tag_sha=$$(git rev-parse "$$latest_tag") && \ - printf '%-14s = %s\n' 'Latest tag' "$$latest_tag" \ + printf '%-14s = %s\n' 'VERSION file' "$$version_file" \ + 'changes.rst' "$$changes_file" \ + 'Latest tag' "$$latest_tag" \ 'HEAD SHA' "$$head_sha" \ 'Latest tag SHA' "$$latest_tag_sha" && \ + test "$$version_file" = "$$changes_file" && \ + test "$$latest_tag" = "$$version_file" && \ test "$$head_sha" = "$$latest_tag_sha" make force_release From 5cf7f9777bc3f5bb92c57116071fd2bed3ac0b70 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 09:05:36 -0400 Subject: [PATCH 0602/1790] Fix non-venv branch always failing cc202cc put an end to the problem where, when run outside a virtual environment, it would always "succeed", because all "|| echo ..." required to succeed was for the echo command reporting the error to succeed. Unfortunately, that commit created the oppposite problem, causing that case to always fail! This commit fixes it, so it fails when there is an error, and succeeds when there is no error. --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 52d080788..fdefb0439 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,7 @@ force_release: clean python -m build --sdist --wheel; \ else \ python3 -m build --sdist --wheel || \ - echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" && \ - false; \ + { echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" && false; }; \ fi # Upload to PyPI and push the tag. From 35f233949ec0cef92ce4d51129de18f358f8f525 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Sep 2023 08:48:45 +0200 Subject: [PATCH 0603/1790] delete sublime-text project, nobody uses it and it's probably very outdated --- etc/sublime-text/git-python.sublime-project | 62 --------------------- 1 file changed, 62 deletions(-) delete mode 100644 etc/sublime-text/git-python.sublime-project diff --git a/etc/sublime-text/git-python.sublime-project b/etc/sublime-text/git-python.sublime-project deleted file mode 100644 index 3dab9f656..000000000 --- a/etc/sublime-text/git-python.sublime-project +++ /dev/null @@ -1,62 +0,0 @@ -{ - "folders": - [ - // GIT-PYTHON - ///////////// - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "git/ext", - "dist", - ".tox", - "doc/build", - "*.egg-info" - ] - }, - // GITDB - //////// - { - "follow_symlinks": true, - "path": "../../git/ext/gitdb", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "gitdb/ext", - "dist", - "doc/build", - ".tox", - ] - }, - // // SMMAP - // //////// - { - "follow_symlinks": true, - "path": "../../git/ext/gitdb/gitdb/ext/smmap", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - ] - }, - ] -} From 6495d84142b60ba81a5b4268a0dfc0785c22d60a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 04:24:17 -0400 Subject: [PATCH 0604/1790] Extract checks from release target to script This extracts the check logic from the release target in Makefile to a new script, check-version.sh. The code is also modified, mainly to account for different ways output is displayed and errors are reported and treated in a Makefile versus a standalone shell script. (The .sh suffix is for consistency with the naming of init-tests-after-clone.sh and is *not* intended to suggest sourcing the script; this script should be executed, not sourced.) --- .gitattributes | 1 + Makefile | 26 +------------------------- check-version.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 25 deletions(-) create mode 100755 check-version.sh diff --git a/.gitattributes b/.gitattributes index 739b2be29..eb503040b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ test/fixtures/* eol=lf init-tests-after-clone.sh eol=lf +check-version.sh eol=lf Makefile eol=lf diff --git a/Makefile b/Makefile index fdefb0439..d11dd4de9 100644 --- a/Makefile +++ b/Makefile @@ -7,31 +7,7 @@ clean: rm -rf build/ dist/ .eggs/ .tox/ release: clean - # Check that VERSION and changes.rst exist and have no uncommitted changes - test -f VERSION - test -f doc/source/changes.rst - git status -s VERSION doc/source/changes.rst - @test -z "$$(git status -s VERSION doc/source/changes.rst)" - - # Check that ALL changes are commited (can comment out if absolutely necessary) - git status -s - @test -z "$$(git status -s)" - - # Check that latest tag matches version and is the current head we're releasing - @version_file="$$(cat VERSION)" && \ - changes_file="$$(awk '/^[0-9]/ {print $$0; exit}' doc/source/changes.rst)" && \ - config_opts="$$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" && \ - latest_tag=$$(git $$config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1) && \ - head_sha=$$(git rev-parse HEAD) latest_tag_sha=$$(git rev-parse "$$latest_tag") && \ - printf '%-14s = %s\n' 'VERSION file' "$$version_file" \ - 'changes.rst' "$$changes_file" \ - 'Latest tag' "$$latest_tag" \ - 'HEAD SHA' "$$head_sha" \ - 'Latest tag SHA' "$$latest_tag_sha" && \ - test "$$version_file" = "$$changes_file" && \ - test "$$latest_tag" = "$$version_file" && \ - test "$$head_sha" = "$$latest_tag_sha" - + ./check-version.sh make force_release force_release: clean diff --git a/check-version.sh b/check-version.sh new file mode 100755 index 000000000..5d3157033 --- /dev/null +++ b/check-version.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# This script checks if we appear ready to build and publish a new release. +# See the release instructions in README.md for the steps to make this pass. + +set -eEfuo pipefail +trap 'printf "%s: Check failed. Stopping.\n" "$0" >&2' ERR + +readonly version_path='VERSION' +readonly changes_path='doc/source/changes.rst' + +printf 'Checking current directory.\n' +test "$(cd -- "$(dirname -- "$0")" && pwd)" = "$(pwd)" # Ugly, but portable. + +printf 'Checking that %s and %s exist and have no committed changes.\n' \ + "$version_path" "$changes_path" +test -f "$version_path" +test -f "$changes_path" +git status -s -- "$version_path" "$changes_path" +test -z "$(git status -s -- "$version_path" "$changes_path")" + +# This section can be commented out, if absolutely necessary. +printf 'Checking that ALL changes are committed.\n' +git status -s +test -z "$(git status -s)" + +printf 'Gathering current version, latest tag, and current HEAD commit info.\n' +version_version="$(cat "$version_path")" +changes_version="$(awk '/^[0-9]/ {print $0; exit}' "$changes_path")" +config_opts="$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" +latest_tag="$(git $config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1)" +head_sha="$(git rev-parse HEAD)" +latest_tag_sha="$(git rev-parse "$latest_tag")" + +# Display a table of all the current version, tag, and HEAD commit information. +printf '%-14s = %s\n' 'VERSION file' "$version_version" \ + 'changes.rst' "$changes_version" \ + 'Latest tag' "$latest_tag" \ + 'HEAD SHA' "$head_sha" \ + 'Latest tag SHA' "$latest_tag_sha" + +# Check that latest tag matches version and is the current HEAD we're releasing +test "$version_version" = "$changes_version" +test "$latest_tag" = "$version_version" +test "$head_sha" = "$latest_tag_sha" +printf 'OK, everything looks good.\n' From 4b1c56409e905b852e3c93de142e109b147eee5e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 05:38:24 -0400 Subject: [PATCH 0605/1790] Extract build from force_release target to script This moves the conditional build dependency installation logic and build logic from the force_release Makefile target to a shell script build-release.sh, which force_release calls. The code is changed to clean it up, and also to account for differences between how output is displayed and errors reported in Makefiles and shell scripts. (As in check-version.sh, the .sh suffix does not signify anything about how the script is to be used: like the other shell scripts in the project, this should be executed, no sourced.) --- .gitattributes | 3 ++- Makefile | 13 +------------ build-release.sh | 22 ++++++++++++++++++++++ check-version.sh | 3 ++- 4 files changed, 27 insertions(+), 14 deletions(-) create mode 100755 build-release.sh diff --git a/.gitattributes b/.gitattributes index eb503040b..a66dc90ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,5 @@ test/fixtures/* eol=lf init-tests-after-clone.sh eol=lf -check-version.sh eol=lf Makefile eol=lf +check-version.sh eol=lf +build-release.sh eol=lf diff --git a/Makefile b/Makefile index d11dd4de9..38090244c 100644 --- a/Makefile +++ b/Makefile @@ -11,17 +11,6 @@ release: clean make force_release force_release: clean - # IF we're in a virtual environment, add build tools - test -z "$$VIRTUAL_ENV" || pip install -U build twine - - # Build the sdist and wheel that will be uploaded to PyPI. - if test -n "$$VIRTUAL_ENV"; then \ - python -m build --sdist --wheel; \ - else \ - python3 -m build --sdist --wheel || \ - { echo "Use a virtual-env with 'python -m venv env && source env/bin/activate' instead" && false; }; \ - fi - - # Upload to PyPI and push the tag. + ./build-release.sh twine upload dist/* git push --tags origin main diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 000000000..ebb45e062 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# This script builds a release. If run in a venv, it auto-installs its tools. +# You may want to run "make release" instead of running this script directly. + +set -eEu + +if test -n "${VIRTUAL_ENV:-}"; then + deps=(build twine) # Install twine along with build, as we need it later. + printf 'Virtual environment detected. Adding packages: %s\n' "${deps[*]}" + pip install -U "${deps[@]}" + printf 'Starting the build.\n' + python -m build --sdist --wheel +else + suggest_venv() { + venv_cmd='python -m venv env && source env/bin/activate' + printf "Use a virtual-env with '%s' instead.\n" "$venv_cmd" + } + trap suggest_venv ERR # This keeps the original exit (error) code. + printf 'Starting the build.\n' + python3 -m build --sdist --wheel # Outside a venv, use python3. +fi diff --git a/check-version.sh b/check-version.sh index 5d3157033..802492d93 100755 --- a/check-version.sh +++ b/check-version.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # -# This script checks if we appear ready to build and publish a new release. +# This script checks if we are in a consistent state to build a new release. # See the release instructions in README.md for the steps to make this pass. +# You may want to run "make release" instead of running this script directly. set -eEfuo pipefail trap 'printf "%s: Check failed. Stopping.\n" "$0" >&2' ERR From 729372f6f87639f0c4d8211ee7d173100117a257 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 07:16:08 -0400 Subject: [PATCH 0606/1790] Prevent buggy interaction between MinGW and WSL This changes the hashbangs in Makefile helper scripts to be static. Often, "#!/usr/bin/env bash" is a better hashbang for bash scripts than "#!/bin/bash", because it automatically works on Unix-like systems that are not GNU/Linux and do not have bash in /bin, but on which it has been installed in another $PATH directory, such as /usr/local/bin. (It can also be helpful on macOS, where /bin/bash is usually an old version of bash, while a package manager such as brew may have been used to install a newer version elsewhere.) Windows systems with WSL installed often have a deprecated bash.exe in the System32 directory that runs commands and scripts inside an installed WSL system. (wsl.exe should be used instead.) Anytime that bash is used due to a "#!/usr/bin/env bash" hashbang, it is wrong, because that only happens if the caller is some Unix-style script running natively or otherwise outside WSL. Normally this is not a reason to prefer a "#!/bin/bash" hashbang, because normally any environment in which one can run a script in a way that determines its interpreter from its hashbang is an environment in which a native (or otherwise appropriate) bash precedes the System32 bash in a PATH search. However, MinGW make, a popular make implementation used on Windows, is an exception. The goal of this change is not to sacrifice support for some Unix-like systems to better support Windows, which wouldn't necessarily be justified. Rather, this is to avoid extremely confusing wrong behavior that in some cases would have bizarre effects that are very hard to detect. I discovered this problem because the VIRTUAL_ENV variable was not inheried by the bash interpreter (because it was, fortunately, not passed through to WSL). But if "python3 -m build" finds a global "build" package, things might get much further before any problem is noticed. --- build-release.sh | 2 +- check-version.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-release.sh b/build-release.sh index ebb45e062..cbf0e91a9 100755 --- a/build-release.sh +++ b/build-release.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # # This script builds a release. If run in a venv, it auto-installs its tools. # You may want to run "make release" instead of running this script directly. diff --git a/check-version.sh b/check-version.sh index 802492d93..e74ec2606 100755 --- a/check-version.sh +++ b/check-version.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # # This script checks if we are in a consistent state to build a new release. # See the release instructions in README.md for the steps to make this pass. From ba84db487a31c593fe0618a63f80709e405039c9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 08:11:42 -0400 Subject: [PATCH 0607/1790] Fix message wording that was opposite of intended This also makes a correct but confusing comment clearer. --- check-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check-version.sh b/check-version.sh index e74ec2606..373403c0e 100755 --- a/check-version.sh +++ b/check-version.sh @@ -13,7 +13,7 @@ readonly changes_path='doc/source/changes.rst' printf 'Checking current directory.\n' test "$(cd -- "$(dirname -- "$0")" && pwd)" = "$(pwd)" # Ugly, but portable. -printf 'Checking that %s and %s exist and have no committed changes.\n' \ +printf 'Checking that %s and %s exist and have no uncommitted changes.\n' \ "$version_path" "$changes_path" test -f "$version_path" test -f "$changes_path" @@ -40,7 +40,7 @@ printf '%-14s = %s\n' 'VERSION file' "$version_version" \ 'HEAD SHA' "$head_sha" \ 'Latest tag SHA' "$latest_tag_sha" -# Check that latest tag matches version and is the current HEAD we're releasing +# Check that the latest tag and current version match the HEAD we're releasing. test "$version_version" = "$changes_version" test "$latest_tag" = "$version_version" test "$head_sha" = "$latest_tag_sha" From de40e6864d5cbf8cd604b6f718b876c4d8c5a323 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 08:57:34 -0400 Subject: [PATCH 0608/1790] Ignore some other virtual environment directories Like ".venv" and "venv", ".env" and "env" are common, plus "env" appears in the example command shown for making a virtual environment for the purpose of building a release, under some circumstances when "make release" or "make force_release" fail. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 72da84eee..0bd307639 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.py[co] *.swp *~ +.env/ +env/ .venv/ venv/ /*.egg-info From 693d041869497137085171cdabbaf43e33fb9c84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Sep 2023 07:40:36 +0200 Subject: [PATCH 0609/1790] make `.gitattributes` file more generic That way shell scripts will be handled correctly by default, anywhere. --- .gitattributes | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index a66dc90ca..3f3d2f050 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,3 @@ test/fixtures/* eol=lf -init-tests-after-clone.sh eol=lf -Makefile eol=lf -check-version.sh eol=lf -build-release.sh eol=lf +*.sh eol=lf +/Makefile eol=lf From 962f747d9c2a877e70886f2ebee975b9be4bb672 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Sep 2023 07:47:31 +0200 Subject: [PATCH 0610/1790] submodules don't contribute to the release; ignore their changes --- check-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check-version.sh b/check-version.sh index 373403c0e..81112b326 100755 --- a/check-version.sh +++ b/check-version.sh @@ -22,8 +22,8 @@ test -z "$(git status -s -- "$version_path" "$changes_path")" # This section can be commented out, if absolutely necessary. printf 'Checking that ALL changes are committed.\n' -git status -s -test -z "$(git status -s)" +git status -s --ignore-submodules +test -z "$(git status -s --ignore-submodules)" printf 'Gathering current version, latest tag, and current HEAD commit info.\n' version_version="$(cat "$version_path")" From d18d90a2c04abeff4bcc8d642fcd33be1c1eb35b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Sep 2023 07:53:55 +0200 Subject: [PATCH 0611/1790] Use 'echo' where possible to avoid explicit newlines --- check-version.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/check-version.sh b/check-version.sh index 81112b326..d29d457c0 100755 --- a/check-version.sh +++ b/check-version.sh @@ -5,27 +5,26 @@ # You may want to run "make release" instead of running this script directly. set -eEfuo pipefail -trap 'printf "%s: Check failed. Stopping.\n" "$0" >&2' ERR +trap 'echo "$0: Check failed. Stopping." >&2' ERR readonly version_path='VERSION' readonly changes_path='doc/source/changes.rst' -printf 'Checking current directory.\n' +echo 'Checking current directory.' test "$(cd -- "$(dirname -- "$0")" && pwd)" = "$(pwd)" # Ugly, but portable. -printf 'Checking that %s and %s exist and have no uncommitted changes.\n' \ - "$version_path" "$changes_path" +echo "Checking that $version_path and $changes_path exist and have no uncommitted changes." test -f "$version_path" test -f "$changes_path" git status -s -- "$version_path" "$changes_path" test -z "$(git status -s -- "$version_path" "$changes_path")" # This section can be commented out, if absolutely necessary. -printf 'Checking that ALL changes are committed.\n' +echo 'Checking that ALL changes are committed.' git status -s --ignore-submodules test -z "$(git status -s --ignore-submodules)" -printf 'Gathering current version, latest tag, and current HEAD commit info.\n' +echo 'Gathering current version, latest tag, and current HEAD commit info.' version_version="$(cat "$version_path")" changes_version="$(awk '/^[0-9]/ {print $0; exit}' "$changes_path")" config_opts="$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" @@ -44,4 +43,4 @@ printf '%-14s = %s\n' 'VERSION file' "$version_version" \ test "$version_version" = "$changes_version" test "$latest_tag" = "$version_version" test "$head_sha" = "$latest_tag_sha" -printf 'OK, everything looks good.\n' +echo 'OK, everything looks good.' From 5919f8d04bccfaf0c98ae032437635d1a2de656b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Sep 2023 08:06:51 +0200 Subject: [PATCH 0612/1790] Be explicit on how to interpret the data table --- check-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-version.sh b/check-version.sh index d29d457c0..6d10d6785 100755 --- a/check-version.sh +++ b/check-version.sh @@ -24,7 +24,6 @@ echo 'Checking that ALL changes are committed.' git status -s --ignore-submodules test -z "$(git status -s --ignore-submodules)" -echo 'Gathering current version, latest tag, and current HEAD commit info.' version_version="$(cat "$version_path")" changes_version="$(awk '/^[0-9]/ {print $0; exit}' "$changes_path")" config_opts="$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" @@ -33,6 +32,7 @@ head_sha="$(git rev-parse HEAD)" latest_tag_sha="$(git rev-parse "$latest_tag")" # Display a table of all the current version, tag, and HEAD commit information. +echo $'\nThe VERSION must be the same in all locations, and so must the HEAD and tag SHA' printf '%-14s = %s\n' 'VERSION file' "$version_version" \ 'changes.rst' "$changes_version" \ 'Latest tag' "$latest_tag" \ From 1e0b3f91f6dffad6bfc262528c14bf459c6a63c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Sep 2023 08:32:30 +0200 Subject: [PATCH 0613/1790] refinements to `build-reelase.sh` - use `echo` where feasible to avoid explicit newlines - use `function` syntax out of habit - deduplicate release invocation - make `venv` based invocation less verbose - make help-text in non-venv more prominent --- build-release.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/build-release.sh b/build-release.sh index cbf0e91a9..5840e4472 100755 --- a/build-release.sh +++ b/build-release.sh @@ -5,18 +5,22 @@ set -eEu +function release_with() { + $1 -m build --sdist --wheel +} + if test -n "${VIRTUAL_ENV:-}"; then deps=(build twine) # Install twine along with build, as we need it later. - printf 'Virtual environment detected. Adding packages: %s\n' "${deps[*]}" - pip install -U "${deps[@]}" - printf 'Starting the build.\n' - python -m build --sdist --wheel + echo "Virtual environment detected. Adding packages: ${deps[*]}" + pip install --quiet --upgrade "${deps[@]}" + echo 'Starting the build.' + release_with python else - suggest_venv() { + function suggest_venv() { venv_cmd='python -m venv env && source env/bin/activate' - printf "Use a virtual-env with '%s' instead.\n" "$venv_cmd" + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" } trap suggest_venv ERR # This keeps the original exit (error) code. - printf 'Starting the build.\n' - python3 -m build --sdist --wheel # Outside a venv, use python3. + echo 'Starting the build.' + release_with python3 # Outside a venv, use python3. fi From 810ae3ad3f94a6a1b54231f88d13b7e071d1278c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 10:22:18 +0200 Subject: [PATCH 0614/1790] prepare v6.0.0 --- doc/source/changes.rst | 7 +++++++ smmap/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d0623b533..b77de7940 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +****** +v6.0.0 +****** + +- Dropped support 3.6 and 3.7 +- Declared support for Python 3.11 and 3.12 + ****** v5.0.0 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 6b1a44cb5..5f37edae4 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (5, 0, 0) +version_info = (6, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From ce03fde1a0059f6d7db9cf40b9a9d12b75797e7b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 10:25:42 +0200 Subject: [PATCH 0615/1790] improve release setup based on the one in GitPython --- .gitignore | 1 + Makefile | 51 +++++++----------------------------------------- build-release.sh | 26 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 44 deletions(-) create mode 100755 build-release.sh diff --git a/.gitignore b/.gitignore index 01247547d..85cc74fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ MANIFEST *.egg-info .noseids *.sublime-workspace +/env/ diff --git a/Makefile b/Makefile index 051e99ba0..978632e66 100644 --- a/Makefile +++ b/Makefile @@ -1,49 +1,12 @@ -.PHONY: build sdist cover test clean-files clean-docs doc all +.PHONY: all clean release force_release all: - $(info Possible targets:) - $(info doc) - $(info clean-docs) - $(info clean-files) - $(info clean) - $(info test) - $(info coverage) - $(info build) - $(info sdist) + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all -doc: - cd doc && make html +clean: + rm -rf build/ dist/ .eggs/ .tox/ -clean-docs: - cd doc && make clean - -clean-files: - git clean -fx - rm -rf build/ dist/ - -clean: clean-files clean-docs - -test: - pytest - -coverage: - pytest --cov smmap --cov-report xml - -build: - ./setup.py build - -sdist: - ./setup.py sdist - -release: clean - # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag | sort -nr | head -n1)" - echo "HEAD SHA = $$(git rev-parse head)" - echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - make force_release - -force_release:: clean - git push --tags - python3 setup.py sdist bdist_wheel +force_release: clean + ./build-release.sh twine upload dist/* + git push --tags origin main diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 000000000..5840e4472 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This script builds a release. If run in a venv, it auto-installs its tools. +# You may want to run "make release" instead of running this script directly. + +set -eEu + +function release_with() { + $1 -m build --sdist --wheel +} + +if test -n "${VIRTUAL_ENV:-}"; then + deps=(build twine) # Install twine along with build, as we need it later. + echo "Virtual environment detected. Adding packages: ${deps[*]}" + pip install --quiet --upgrade "${deps[@]}" + echo 'Starting the build.' + release_with python +else + function suggest_venv() { + venv_cmd='python -m venv env && source env/bin/activate' + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" + } + trap suggest_venv ERR # This keeps the original exit (error) code. + echo 'Starting the build.' + release_with python3 # Outside a venv, use python3. +fi From e163592304fbe1015605270cfdf4e3529904a986 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 10:30:54 +0200 Subject: [PATCH 0616/1790] adjust force-release target to handle differently named main branch --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 978632e66..20436bbc4 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,4 @@ clean: force_release: clean ./build-release.sh twine upload dist/* - git push --tags origin main + git push --tags origin master From b98fdd1fad62c2a957ba76dee5361d55a0cba14c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 04:49:58 -0400 Subject: [PATCH 0617/1790] Revert "Drop support for EOL Python 3.7" Analogous to https://github.com/gitpython-developers/gitdb/pull/94, this brings back Python 3.7 support (allowing it to be installed on 3.7, and testing on 3.7), even though 3.7 is end of life, because support for 3.7 is not being dropped by GitPython (or gitdb) yet, and there is value in keeping GitPython, gitdb, and smmap consistent. This reverts commit f1b4d14e7d3902135af336ed06a12308dfd40270. --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 ++- tox.ini | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 55c654269..73613588d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index ff973bae6..e21f53453 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.8+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8f36481c2..109fec247 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.8+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index b99532814..49d90a02d 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.8", + python_requires=">=3.7", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,6 +38,7 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tox.ini b/tox.ini index ac3536caa..f7205c5a3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{38, 39, 310, 311, 312} +envlist = flake8, py{37, 38, 39, 310, 311, 312} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 256c5a21de2d14aca02c9689d7d63f78c4e0ef61 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 13:31:58 +0200 Subject: [PATCH 0618/1790] prepare v5.0.1 --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index b77de7940..947c71a81 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,10 +2,18 @@ Changelog ######### +****** +v5.0.1 +****** + +- Added support for Python 3.12 + ****** v6.0.0 ****** +YANKED + - Dropped support 3.6 and 3.7 - Declared support for Python 3.11 and 3.12 diff --git a/smmap/__init__.py b/smmap/__init__.py index 5f37edae4..429f47aed 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (6, 0, 0) +version_info = (5, 0, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From ae8c018f3ab62f7ada3a56af29d4f647809654c6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 08:07:23 -0400 Subject: [PATCH 0619/1790] Fix URLs that were redirecting to another license All the opensource.org BSD license URLs at the top of source code files in this project had originally pointed to a page on the 3-clause BSD license that this project used and continues to use. But over time the site was apparently reorganized and the link became a redirect to the page about the 2-clause BSD license. Because it is identified only as the "BSD license" in the comments in this project that contain the links, this unfortunately makes it so those top-of-file comments all wrongly claim that the project is 2-clause BSD licensed. This fixes the links by replacing them with the current URL of the opensource.org page on the 3-clause BSD license. The current URL contains "bsd-3-clause" in it, so this specific problem is unlikely to recur with that URL (and even if it did, the text "bsd-3-clause is information that may clue readers in to what is going on). --- git/__init__.py | 2 +- git/cmd.py | 2 +- git/compat.py | 2 +- git/config.py | 2 +- git/diff.py | 2 +- git/exc.py | 2 +- git/index/base.py | 2 +- git/objects/base.py | 2 +- git/objects/blob.py | 2 +- git/objects/commit.py | 2 +- git/objects/tag.py | 2 +- git/objects/tree.py | 2 +- git/objects/util.py | 2 +- git/remote.py | 2 +- git/repo/base.py | 2 +- git/types.py | 2 +- git/util.py | 2 +- test/__init__.py | 2 +- test/lib/__init__.py | 2 +- test/lib/helper.py | 2 +- test/performance/test_commit.py | 2 +- test/test_actor.py | 2 +- test/test_base.py | 2 +- test/test_blob.py | 2 +- test/test_clone.py | 2 +- test/test_commit.py | 2 +- test/test_config.py | 2 +- test/test_db.py | 2 +- test/test_diff.py | 5 ++--- test/test_docs.py | 2 +- test/test_exc.py | 2 +- test/test_git.py | 2 +- test/test_index.py | 4 ++-- test/test_installation.py | 2 +- test/test_refs.py | 2 +- test/test_remote.py | 2 +- test/test_repo.py | 2 +- test/test_stats.py | 2 +- test/test_submodule.py | 2 +- test/test_tree.py | 2 +- test/test_util.py | 2 +- 41 files changed, 43 insertions(+), 44 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 6196a42d7..e2d123fa5 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ # flake8: noqa # @PydevCodeAnalysisIgnore from git.exc import * # @NoMove @IgnorePep8 diff --git a/git/cmd.py b/git/cmd.py index d6f8f946a..9921dd6c9 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from __future__ import annotations import re import contextlib diff --git a/git/compat.py b/git/compat.py index e7ef28c30..624f26116 100644 --- a/git/compat.py +++ b/git/compat.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ """utilities to help provide compatibility with python 3""" # flake8: noqa diff --git a/git/config.py b/git/config.py index 1973111eb..880eb9301 100644 --- a/git/config.py +++ b/git/config.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing module parser implementation able to properly read and write configuration files""" diff --git a/git/diff.py b/git/diff.py index 1424ff3ad..3e3de7bc1 100644 --- a/git/diff.py +++ b/git/diff.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import re from git.cmd import handle_process_output diff --git a/git/exc.py b/git/exc.py index 775528bf6..0786a8e8a 100644 --- a/git/exc.py +++ b/git/exc.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ """ Module containing all exceptions thrown throughout the git package, """ from gitdb.exc import BadName # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 diff --git a/git/index/base.py b/git/index/base.py index 193baf3ad..cf016df6a 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from contextlib import ExitStack import datetime diff --git a/git/objects/base.py b/git/objects/base.py index eb9a8ac3d..1d07fd0f6 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from git.exc import WorkTreeRepositoryUnsupported from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex diff --git a/git/objects/blob.py b/git/objects/blob.py index 1881f210c..96ce486f5 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from mimetypes import guess_type from . import base diff --git a/git/objects/commit.py b/git/objects/commit.py index 6db3ea0f3..88c485d09 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import datetime import re from subprocess import Popen, PIPE diff --git a/git/objects/tag.py b/git/objects/tag.py index 3956a89e7..56fd05d1a 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ """ Module containing all object based types. """ from . import base from .util import get_object_type_by_name, parse_actor_and_date diff --git a/git/objects/tree.py b/git/objects/tree.py index a9b491e23..4f490af54 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from git.util import IterableList, join_path import git.diff as git_diff diff --git a/git/objects/util.py b/git/objects/util.py index 56938507e..992a53d9c 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ """Module for general utility functions""" # flake8: noqa F401 diff --git a/git/remote.py b/git/remote.py index 95a2b8ac6..fc2b2ceba 100644 --- a/git/remote.py +++ b/git/remote.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ # Module implementing a remote object allowing easy access to git remotes import logging diff --git a/git/repo/base.py b/git/repo/base.py index 113fca459..fda3cdc88 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from __future__ import annotations import logging import os diff --git a/git/types.py b/git/types.py index 9f8621721..21276b5f1 100644 --- a/git/types.py +++ b/git/types.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ # flake8: noqa import os diff --git a/git/util.py b/git/util.py index 636e79806..638807a36 100644 --- a/git/util.py +++ b/git/util.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from abc import abstractmethod import os.path as osp diff --git a/test/__init__.py b/test/__init__.py index 757cbad1f..a3d514523 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -2,4 +2,4 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/lib/__init__.py b/test/lib/__init__.py index a4e57b8e0..299317c0b 100644 --- a/test/lib/__init__.py +++ b/test/lib/__init__.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ # flake8: noqa import inspect diff --git a/test/lib/helper.py b/test/lib/helper.py index c04c5cd90..64c70a8f4 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib from functools import wraps import gc diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index 38b529af7..dbe2ad43e 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO from time import time import sys diff --git a/test/test_actor.py b/test/test_actor.py index ce0c74fc9..f495ac084 100644 --- a/test/test_actor.py +++ b/test/test_actor.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from test.lib import TestBase from git import Actor diff --git a/test/test_base.py b/test/test_base.py index 30029367d..b77c8117d 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import os import sys import tempfile diff --git a/test/test_blob.py b/test/test_blob.py index b94dcec23..692522b52 100644 --- a/test/test_blob.py +++ b/test/test_blob.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from test.lib import TestBase from git import Blob diff --git a/test/test_clone.py b/test/test_clone.py index 304ab33cb..1b4a6c332 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from pathlib import Path import re diff --git a/test/test_commit.py b/test/test_commit.py index 4871902ec..d13db1410 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import copy from datetime import datetime from io import BytesIO diff --git a/test/test_config.py b/test/test_config.py index b159ebe2d..481e129c6 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import glob import io diff --git a/test/test_db.py b/test/test_db.py index 228c70e7c..ebf73b535 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from git.db import GitCmdObjectDB from git.exc import BadObject from test.lib import TestBase diff --git a/test/test_diff.py b/test/test_diff.py index 504337744..dacbdc3bc 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import ddt import shutil import tempfile @@ -414,7 +414,7 @@ def test_diff_interface(self): @with_rw_directory def test_rename_override(self, rw_dir): - """Test disabling of diff rename detection""" + """Test disabling of diff rename detection""" # create and commit file_a.txt repo = Repo.init(rw_dir) @@ -480,4 +480,3 @@ def test_rename_override(self, rw_dir): self.assertEqual(True, diff.renamed_file) self.assertEqual('file_a.txt', diff.rename_from) self.assertEqual('file_b.txt', diff.rename_to) - diff --git a/test/test_docs.py b/test/test_docs.py index 20027c191..505b50f77 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import os import sys diff --git a/test/test_exc.py b/test/test_exc.py index f998ff4d5..9e125d246 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009, 2016 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import re diff --git a/test/test_git.py b/test/test_git.py index 4d57a2d86..2c392155a 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import os import shutil import subprocess diff --git a/test/test_index.py b/test/test_index.py index 3bebb382b..9b7ba52a6 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO import os @@ -953,4 +953,4 @@ def test_index_add_pathlike(self, rw_repo): file = git_dir / "file.txt" file.touch() - rw_repo.index.add(file) \ No newline at end of file + rw_repo.index.add(file) diff --git a/test/test_installation.py b/test/test_installation.py index d856ebc94..1c9d2359c 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -1,5 +1,5 @@ # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import ast import os diff --git a/test/test_refs.py b/test/test_refs.py index e7526c3b2..afd273df9 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from itertools import chain from pathlib import Path diff --git a/test/test_remote.py b/test/test_remote.py index 9636ca486..e0dcb4131 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import random import tempfile diff --git a/test/test_repo.py b/test/test_repo.py index 08ed13a00..abae5ad78 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -3,7 +3,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import glob import io from io import BytesIO diff --git a/test/test_stats.py b/test/test_stats.py index 1f6896555..335ce483b 100644 --- a/test/test_stats.py +++ b/test/test_stats.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from test.lib import TestBase, fixture from git import Stats diff --git a/test/test_submodule.py b/test/test_submodule.py index 8c98a671e..5a7f26207 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib import os import shutil diff --git a/test/test_tree.py b/test/test_tree.py index 22c9c7d78..e59705645 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO from unittest import skipIf diff --git a/test/test_util.py b/test/test_util.py index c17efce35..517edd65c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ import os import pickle From e0769d1ed2d9044d7523c2eb2f8a0d44a90deb9e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 08:36:34 -0400 Subject: [PATCH 0620/1790] Fix top-of-file license URLs here in gitdb too This is the gitdb part of the fix for the top-of-file license URLs that have come to point to a page about a related but different license from the one GitPython and gitdb are (intended to be) offered under. See https://github.com/gitpython-developers/GitPython/pull/1662 for details about the problem and how it came about. --- gitdb/__init__.py | 2 +- gitdb/base.py | 2 +- gitdb/db/__init__.py | 2 +- gitdb/db/base.py | 2 +- gitdb/db/git.py | 2 +- gitdb/db/loose.py | 2 +- gitdb/db/mem.py | 2 +- gitdb/db/pack.py | 2 +- gitdb/db/ref.py | 2 +- gitdb/exc.py | 2 +- gitdb/fun.py | 2 +- gitdb/pack.py | 6 +++--- gitdb/stream.py | 10 +++++----- gitdb/test/__init__.py | 2 +- gitdb/test/db/__init__.py | 2 +- gitdb/test/db/lib.py | 2 +- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_mem.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 2 +- gitdb/test/performance/lib.py | 2 +- gitdb/test/performance/test_pack.py | 2 +- gitdb/test/performance/test_pack_streaming.py | 2 +- gitdb/test/performance/test_stream.py | 2 +- gitdb/test/test_base.py | 2 +- gitdb/test/test_example.py | 2 +- gitdb/test/test_pack.py | 4 ++-- gitdb/test/test_stream.py | 2 +- gitdb/test/test_util.py | 2 +- gitdb/typ.py | 2 +- gitdb/util.py | 2 +- 33 files changed, 40 insertions(+), 40 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index b777632bb..2fb3f7edb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Initialize the object database module""" import sys diff --git a/gitdb/base.py b/gitdb/base.py index 42e71d0fa..9a23a4f71 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with basic data structures - they are designed to be lightweight and fast""" from gitdb.util import bin_to_hex diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index 0a2a46a64..20fd2280d 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import * from gitdb.db.loose import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index e89052e88..7312fe00f 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains implementations of database retrieveing objects""" from gitdb.util import ( join, diff --git a/gitdb/db/git.py b/gitdb/db/git.py index e2cb46805..a1ed14280 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import ( CompoundDB, ObjectDBW, diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 7ea6fef3d..256fec9cf 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import ( FileDBBase, ObjectDBR, diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1b954cba2..d4772fdb5 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains the MemoryDatabase implementation""" from gitdb.db.loose import LooseObjectDB from gitdb.db.base import ( diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 1ce786b79..274ea5990 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing a database to deal with packs""" from gitdb.db.base import ( FileDBBase, diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 6bb2a64dd..bd3015602 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import codecs from gitdb.db.base import ( CompoundDB, diff --git a/gitdb/exc.py b/gitdb/exc.py index 947e5d8bf..73970372b 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with common exceptions""" from gitdb.util import to_hex_sha diff --git a/gitdb/fun.py b/gitdb/fun.py index a4454de4d..a272e5caa 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains basic c-functions which usually contain performance critical code Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" diff --git a/gitdb/pack.py b/gitdb/pack.py index cabce4c9d..e559e113d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains PackIndexFile and PackFile implementations""" import zlib @@ -263,7 +263,7 @@ def __init__(self, indexpath): def close(self): mman.force_map_handle_removal_win(self._indexpath) self._cursor = None - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -528,7 +528,7 @@ def __init__(self, packpath): def close(self): mman.force_map_handle_removal_win(self._packpath) self._cursor = None - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() diff --git a/gitdb/stream.py b/gitdb/stream.py index 1b5426f02..1e0be84fd 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO @@ -140,7 +140,7 @@ def data(self): def close(self): """Close our underlying stream of compressed bytes if this was allowed during initialization :return: True if we closed the underlying stream - :note: can be called safely + :note: can be called safely """ if self._close: if hasattr(self._m, 'close'): @@ -287,11 +287,11 @@ def read(self, size=-1): # if we hit the end of the stream # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. # They are thorough, and I assume it is truly working. - # Why is this logic as convoluted as it is ? Please look at the table in + # Why is this logic as convoluted as it is ? Please look at the table in # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch - # will be the one that works. - # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 8a681e428..03bd406be 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/test/db/__init__.py b/gitdb/test/db/__init__.py index 8a681e428..03bd406be 100644 --- a/gitdb/test/db/__init__.py +++ b/gitdb/test/db/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index b38f1d5e5..408dd8c66 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 6ecf7d7a8..73ac1a08c 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import os from gitdb.test.db.lib import ( TestDBBase, diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 8cc660b8a..295e2eecd 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index eb563c098..882e54fe5 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 4539f4278..bd07906da 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 664aa54ba..0816e649f 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index da59d3b35..8e602342c 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Utilities used in ODB testing""" from gitdb import OStream diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index fa4dd209d..36916ede3 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains library functions""" from gitdb.test.lib import TestBase diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index f034baffd..fc3d3349f 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Performance tests for object store""" from gitdb.test.performance.lib import ( diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index db790f1db..80c798bbc 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Specific test for pack streams only""" from gitdb.test.performance.lib import ( diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 91dc89178..fb10871ae 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Performance data streaming performance""" from gitdb.test.performance.lib import TestBigRepoR diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 519cdfdc1..8fc9e35bf 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" from gitdb.test.lib import ( TestBase, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index cc4d40dd3..3b4c9084b 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with examples from the tutorial section of the docs""" import os from gitdb.test.lib import TestBase diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index f9461975b..e72348228 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test everything about packs reading and writing""" from gitdb.test.lib import ( TestBase, @@ -242,7 +242,7 @@ def rewind_streams(): # END for each info assert count == len(pack_objs) entity.close() - + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 5e2e1bab2..1e7e941d1 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" from gitdb.test.lib import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 3b3165d14..166b33c3a 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" import tempfile import os diff --git a/gitdb/typ.py b/gitdb/typ.py index 98d15f3ec..314db50a7 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing information about types known to the database""" str_blob_type = b'blob' diff --git a/gitdb/util.py b/gitdb/util.py index 3151c061e..bb6d8797a 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import binascii import os import mmap From f1ddf0155ce22041b2db19cdf902e30e0db97c54 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 08:55:21 -0400 Subject: [PATCH 0621/1790] Update ci, in line with gitdb This updates smmap's CI configuration in ways that are in line with recent updates to gitdb's. In most cases there is no difference in the changes, and the reason for the updates is more to avoid confusing differences than from the value of the changes themselves. In one case, there is a major difference (fetch-depth). - https://github.com/gitpython-developers/gitdb/pull/89 (same) - https://github.com/gitpython-developers/gitdb/pull/90 (same) It's just the project, not dependencies, but otherwise the same. - https://github.com/gitpython-developers/gitdb/pull/92 (opposite) This is the major difference. We don't need more than the tip of the branch in these tests. Keeping the default fetch-depth of 1 by not setting it explicitly avoids giving the impression that the tests here are doing something they are not (and also serves as a speed optimization). - https://github.com/gitpython-developers/gitdb/pull/93 (same) --- .github/dependabot.yml | 6 ++++++ .github/workflows/pythonpackage.yml | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..203f3c889 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73613588d..7e21bb9a7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -16,19 +16,23 @@ jobs: strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - experimental: false + - python-version: "3.12" + experimental: true + continue-on-error: ${{ matrix.experimental }} steps: - uses: actions/checkout@v4 - with: - fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Install dependencies + allow-prereleases: ${{ matrix.experimental }} + - name: Install project run: | python -m pip install --upgrade pip + pip install . - name: Lint with flake8 run: | pip install flake8 From e1af18377fd69f9c1007f8abf6ccb95b3c5a6558 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 09:42:40 -0400 Subject: [PATCH 0622/1790] Assorted small fixes/improvements to root dir docs This contains misc. formatting fixes and minor proofreading in the top-level documentation files, plus making the text in the LICENSE section of README.md have links both to external information about the license and to the license file itself. Note that the changes to the license file are just removal of excess whitespace (the extra blank line at the end, and spaces appearing at the end of lines). --- AUTHORS | 1 + CONTRIBUTING.md | 2 +- LICENSE | 35 +++++++++++++++++------------------ README.md | 7 ++++--- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/AUTHORS b/AUTHORS index ba5636db8..3e99ff785 100644 --- a/AUTHORS +++ b/AUTHORS @@ -52,4 +52,5 @@ Contributors are: -Joseph Hale -Santos Gallegos -Wenhan Zhu + Portions derived from other open source works and are clearly marked. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56af0df2a..e108f1b80 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ The following is a short step-by-step rundown of what one typically would do to contribute. -- [fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. +- [Fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. - For setting up the environment to run the self tests, please run `init-tests-after-clone.sh`. - Please try to **write a test that fails unless the contribution is present.** - Try to avoid massive commits and prefer to take small steps, with one commit for each. diff --git a/LICENSE b/LICENSE index 5a9a6f8d3..ba8a219fe 100644 --- a/LICENSE +++ b/LICENSE @@ -1,30 +1,29 @@ Copyright (C) 2008, 2009 Michael Trier and contributors All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* Neither the name of the GitPython project nor the names of -its contributors may be used to endorse or promote products derived +* Neither the name of the GitPython project nor the names of +its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/README.md b/README.md index ca470a851..69d69c56f 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ Please have a look at the [contributions file][contributing]. - [User Documentation](http://gitpython.readthedocs.org) - [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) -- Please post on stackoverflow and use the `gitpython` tag +- Please post on Stack Overflow and use the `gitpython` tag - [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) - Post reproducible bugs and feature requests as a new issue. Please be sure to provide the following information if posting bugs: @@ -267,6 +267,7 @@ gpg --edit-key 4C08421980C9 ### LICENSE -New BSD License. See the LICENSE file. +[New BSD License](https://opensource.org/license/bsd-3-clause/). See the [LICENSE file](https://github.com/gitpython-developers/GitPython/blob/main/license). -[contributing]: https://github.com/gitpython-developers/GitPython/blob/master/CONTRIBUTING.md +[contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md +[license]: https://github.com/gitpython-developers/GitPython/blob/main/license From ad76c99d73ecee3b834beea81ad78665b849adf8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 10:01:38 -0400 Subject: [PATCH 0623/1790] Use venv instead of virtualenv in test_installation This eliminates the test dependency on virtualenv by using the standard library venv module instead in test_installation. --- test-requirements.txt | 1 - test/test_installation.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 62f409824..b00dd6f06 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,4 +6,3 @@ pre-commit pytest pytest-cov pytest-sugar -virtualenv diff --git a/test/test_installation.py b/test/test_installation.py index d856ebc94..6cd97246e 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -4,6 +4,8 @@ import ast import os import subprocess +import sys + from git.compat import is_win from test.lib import TestBase from test.lib.helper import with_rw_directory @@ -12,7 +14,7 @@ class TestInstallation(TestBase): def setUp_venv(self, rw_dir): self.venv = rw_dir - subprocess.run(["virtualenv", self.venv], stdout=subprocess.PIPE) + subprocess.run([sys.executable, "-m", "venv", self.venv], stdout=subprocess.PIPE) bin_name = "Scripts" if is_win else "bin" self.python = os.path.join(self.venv, bin_name, "python") self.pip = os.path.join(self.venv, bin_name, "pip") From 3fbbfd7770c734d2997f16a3f8967ae8f3910dd1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 23:25:19 -0400 Subject: [PATCH 0624/1790] Omit py_modules in setup This removes the py_modules keyword argument in the call to setup, and further shortens/simplifies setup.py by removing the now-unused build_py_modules function. The packages keyword argument already covers this, because we have no loose modules that are included in the distribution, and the call to find_packages: - Omits everything in test/ because it is directed to do so in the call. - Omits the gitdb/ directory (currently existing as a git submodule, not to be confused with Python submodules), because the ext/ directory that contains it does not itself directly contain an __init__.py file, so it is not a traditional package, yet ext/ is contained and found inside the directory git/ that *is* a traditional package, so the ext/ directory is not a namespace package either. - Includes all other modules, recursively, because they are all in a recursive traditional package structure under git/ that find_packages recognizes. To verify that this includes the same files in the built wheel and sdist distributions, I have listed the contents of the wheel with "unzip -l" and the sdist .tar.gz file with "tar tf" both before and after this change, verifying they list all the same entries. --- setup.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/setup.py b/setup.py index bc53bf6c8..90df8d7ea 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from setuptools import setup, find_packages from setuptools.command.build_py import build_py as _build_py from setuptools.command.sdist import sdist as _sdist -import fnmatch import os import sys @@ -62,24 +61,6 @@ def _stamp_version(filename: str) -> None: print("WARNING: Couldn't find version line in file %s" % filename, file=sys.stderr) -def build_py_modules(basedir: str, excludes: Sequence = ()) -> Sequence: - # create list of py_modules from tree - res = set() - _prefix = os.path.basename(basedir) - for root, _, files in os.walk(basedir): - for f in files: - _f, _ext = os.path.splitext(f) - if _ext not in [".py"]: - continue - _f = os.path.join(root, _f) - _f = os.path.relpath(_f, basedir) - _f = "{}.{}".format(_prefix, _f.replace(os.sep, ".")) - if any(fnmatch.fnmatch(_f, x) for x in excludes): - continue - res.add(_f) - return list(res) - - setup( name="GitPython", cmdclass={"build_py": build_py, "sdist": sdist}, @@ -91,7 +72,6 @@ def build_py_modules(basedir: str, excludes: Sequence = ()) -> Sequence: url="https://github.com/gitpython-developers/GitPython", packages=find_packages(exclude=["test", "test.*"]), include_package_data=True, - py_modules=build_py_modules("./git", excludes=["git.ext.*"]), package_dir={"git": "git"}, python_requires=">=3.7", install_requires=requirements, From 407151878361badba63bb809c75cc3a877778de8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Sep 2023 17:57:38 -0400 Subject: [PATCH 0625/1790] Don't track code coverage temporary files While running tests with coverage enabled, files of the form .coverage.MachineName.####.###### are created temporarily. When tests complete normally (whether or not there are failures), these files are automatically removed. However, when tests are cancelled with SIGINT (Ctrl+C), they are sometimes retained. This adds another pattern to .gitignore so that, in addition to not tracking the .coverage file that is retained after tests, these other temporary files are also not tracked. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0bd307639..e8b16da9d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ venv/ /lib/GitPython.egg-info cover/ .coverage +.coverage.* /build /dist /doc/_build From cd052b20a71bec0d605151eeb6b7ac87fbeb3e4a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Sep 2023 03:26:38 -0400 Subject: [PATCH 0626/1790] Start setting up tox It is not completely working yet. --- .gitignore | 1 - requirements-dev.txt | 3 --- tox.ini | 24 ++++++++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 0bd307639..139bf8ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,3 @@ nbproject .pytest_cache/ monkeytype.sqlite3 output.txt -tox.ini diff --git a/requirements-dev.txt b/requirements-dev.txt index f6705341c..e3030c597 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,6 +7,3 @@ flake8-type-checking;python_version>="3.8" # checks for TYPE_CHECKING only pytest-icdiff # pytest-profiling - - -tox diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..0b5591139 --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +requires = tox>=4 +env_list = py{37,38,39,310,311,312}, lint, mypy, black + +[testenv] +description = Run unit tests +package = wheel +extras = test +commands = pytest --color=yes {posargs} + +[testenv:lint] +description = Lint via pre-commit +basepython = py39 +commands = pre-commit run --all-files + +[testenv:mypy] +description = Typecheck with mypy +basepython = py39 +commands = mypy -p git + +[testenv:black] +description = Check style with black +basepython = py39 +commands = black --check --diff git From 2cc2db77574b8197cce215ab703c9d383ea645c1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Sep 2023 19:49:00 -0400 Subject: [PATCH 0627/1790] Pass through SSH_ env vars to tox envs This fixes a problem where the tests for fetching a nonexistent ref prompt like "Enter passphrase for key '/home/USERNAME/.ssh/id_rsa':" and block, if the repository on the machine where the tests are being run has the remote set up using an SSH URL. This passes through all environment variables whose names start with SSH_, even though it should be enough to pass SSH_AGENT_PID and SSH_AUTH_SOCK through, at least for this particular issue. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 0b5591139..9b918f560 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ env_list = py{37,38,39,310,311,312}, lint, mypy, black description = Run unit tests package = wheel extras = test +pass_env = SSH_* commands = pytest --color=yes {posargs} [testenv:lint] From 4bea7cf4cfdbb9d69f24a245bdc8a7e0638524a2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Sep 2023 20:36:31 -0400 Subject: [PATCH 0628/1790] Don't have mypy failure fail the whole tox run Other environments would still be run even after mypy has failed, but to avoid having tox runs be unnecessarily inconsistent with the mypy step in the pythonpackage.yml CI workflow, and also because GitPython is not currently expected to pass mypy checks, this keeps mypy errors from causing the whole tox run to be reported as failed. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9b918f560..a81cd2b45 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ commands = pre-commit run --all-files description = Typecheck with mypy basepython = py39 commands = mypy -p git +ignore_outcome = true [testenv:black] description = Check style with black From e6ec6c87b8ed66e30f7addbd109ab6ec5d74326c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Sep 2023 22:58:58 -0400 Subject: [PATCH 0629/1790] Add tox environment to build HTML documentation The main use of this, similar to the step at the end of pythonpackage.yml, is to find errors produced by building. However, actual documentation *is* built, and unlike other tox environments, running this one actually writes outside the .tox/ directory, creating the documentation in the usual target location. For that reason, this environment is omitted from the env_list, so that it does not run by default and unexpectedly overwrite documentation that may recently have been built before changes are made that could cause generated documentation to be different. --- tox.ini | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index a81cd2b45..8d64b929b 100644 --- a/tox.ini +++ b/tox.ini @@ -11,16 +11,25 @@ commands = pytest --color=yes {posargs} [testenv:lint] description = Lint via pre-commit -basepython = py39 +base_python = py39 commands = pre-commit run --all-files [testenv:mypy] description = Typecheck with mypy -basepython = py39 +base_python = py39 commands = mypy -p git ignore_outcome = true [testenv:black] description = Check style with black -basepython = py39 +base_python = py39 commands = black --check --diff git + +# Run "tox -e html" for this. It is deliberately excluded from env_list, as +# unlike the other environments, this one writes outside the .tox/ directory. +[testenv:html] +description = Build HTML documentation +base_python = py39 +deps = -r doc/requirements.txt +allowlist_externals = make +commands = make -C doc html From a774182c05731797cfe3353ff9618c0cd80b6f35 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Sep 2023 09:02:52 -0400 Subject: [PATCH 0630/1790] Fix black exclusions to omit .gitignore dirs This replaces "exclude" with "extend-exclude" in the black configuration, so that it keeps its default exclusions, of which all directories listed in .gitignore are automatically a part. That makes it possible to run "black ." to format just the files that should be formatted (git/ files, test/ files, and setup.py), while automatically omitting .venv/, .tox/, build/, and so on. This commit does not change how black is run yet, it just fixes the way its exclusions are configured. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42bb31eda..fa06458eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,4 +45,4 @@ omit = ["*/git/ext/*"] [tool.black] line-length = 120 target-version = ['py37'] -exclude = "git/ext/gitdb" +extend-exclude = "git/ext/gitdb" From e39ecb7269fe266311a4bf766c626de5b95a9f9f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Sep 2023 09:06:58 -0400 Subject: [PATCH 0631/1790] Small manual formatting improvements This adds a trailing "," in a few multi-line function calls in test/, where putting one argument per line was intended and is clearer. This is so that when black is run over test/, it recognizes the form and avoids collapsing it. --- test/test_commit.py | 4 ++-- test/test_docs.py | 2 +- test/test_repo.py | 2 +- test/test_submodule.py | 2 +- test/test_util.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_commit.py b/test/test_commit.py index d13db1410..560497547 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -173,12 +173,12 @@ def check_entries(path, changes): ".github/workflows/Future.yml" : { 'insertions': 57, 'deletions': 0, - 'lines': 57 + 'lines': 57, }, ".github/workflows/test_pytest.yml" : { 'insertions': 0, 'deletions': 55, - 'lines': 55 + 'lines': 55, }, } assert path in expected diff --git a/test/test_docs.py b/test/test_docs.py index 505b50f77..4c23e9f81 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -481,7 +481,7 @@ def test_references_and_objects(self, rw_dir): @pytest.mark.xfail( sys.platform == "cygwin", reason="Cygwin GitPython can't find SHA for submodule", - raises=ValueError + raises=ValueError, ) def test_submodules(self): # [1-test_submodules] diff --git a/test/test_repo.py b/test/test_repo.py index abae5ad78..1b46fba7c 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1115,7 +1115,7 @@ def test_repo_odbtype(self): @pytest.mark.xfail( sys.platform == "cygwin", reason="Cygwin GitPython can't find submodule SHA", - raises=ValueError + raises=ValueError, ) def test_submodules(self): self.assertEqual(len(self.rorepo.submodules), 1) # non-recursive diff --git a/test/test_submodule.py b/test/test_submodule.py index 5a7f26207..f7195626f 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -469,7 +469,7 @@ def test_base_bare(self, rwrepo): @pytest.mark.xfail( sys.platform == "cygwin", reason="Cygwin GitPython can't find submodule SHA", - raises=ValueError + raises=ValueError, ) @skipIf( HIDE_WINDOWS_KNOWN_ERRORS, diff --git a/test/test_util.py b/test/test_util.py index 517edd65c..42edc57cf 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -159,7 +159,7 @@ def test_lock_file(self): @pytest.mark.xfail( sys.platform == "cygwin", reason="Cygwin fails here for some reason, always", - raises=AssertionError + raises=AssertionError, ) def test_blocking_lock_file(self): my_file = tempfile.mktemp() From 288cf03e120ed6f7e62d6b0e5c974649e50e69de Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Sep 2023 09:09:08 -0400 Subject: [PATCH 0632/1790] Don't limit black to git/ This changes the documentation in README.md to recommend running "black ." and changes the command to that in tox.ini, so that more paths are covered (in practice, test/ and setup.py). --- README.md | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 69d69c56f..dbec36024 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ mypy -p git For automatic code formatting, run: ```bash -black git +black . ``` Configuration for flake8 is in the `./.flake8` file. diff --git a/tox.ini b/tox.ini index 8d64b929b..82a41e22c 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ ignore_outcome = true [testenv:black] description = Check style with black base_python = py39 -commands = black --check --diff git +commands = black --check --diff . # Run "tox -e html" for this. It is deliberately excluded from env_list, as # unlike the other environments, this one writes outside the .tox/ directory. From 15c736dc79922a1cead221f6fbda5378564e0b6d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Sep 2023 10:20:28 -0400 Subject: [PATCH 0633/1790] Reformat tests with black This actually runs black on the whole project, but the only changes are in test/ (as expected). --- test/performance/test_streams.py | 1 - test/test_commit.py | 17 ++++++++-------- test/test_diff.py | 28 +++++++++++++-------------- test/test_index.py | 2 +- test/test_quick_doc.py | 27 ++++++++++++-------------- test/test_repo.py | 28 ++++++++++++++++----------- test/test_submodule.py | 33 ++++++++++++++++++++++---------- 7 files changed, 75 insertions(+), 61 deletions(-) diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index 5588212e0..25e081578 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -15,7 +15,6 @@ class TestObjDBPerformance(TestBigRepoR): - large_data_size_bytes = 1000 * 1000 * 10 # some MiB should do it moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB diff --git a/test/test_commit.py b/test/test_commit.py index 560497547..f6fb49d50 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -93,7 +93,6 @@ def assert_commit_serialization(self, rwrepo, commit_id, print_performance_info= class TestCommit(TestCommitSerialization): def test_bake(self): - commit = self.rorepo.commit("2454ae89983a4496a445ce347d7a41c0bb0ea7ae") # commits have no dict self.assertRaises(AttributeError, setattr, commit, "someattr", 1) @@ -170,15 +169,15 @@ def test_renames(self): def check_entries(path, changes): expected = { - ".github/workflows/Future.yml" : { - 'insertions': 57, - 'deletions': 0, - 'lines': 57, + ".github/workflows/Future.yml": { + "insertions": 57, + "deletions": 0, + "lines": 57, }, - ".github/workflows/test_pytest.yml" : { - 'insertions': 0, - 'deletions': 55, - 'lines': 55, + ".github/workflows/test_pytest.yml": { + "insertions": 0, + "deletions": 55, + "lines": 55, }, } assert path in expected diff --git a/test/test_diff.py b/test/test_diff.py index dacbdc3bc..9c3888f03 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -419,7 +419,7 @@ def test_rename_override(self, rw_dir): # create and commit file_a.txt repo = Repo.init(rw_dir) file_a = osp.join(rw_dir, "file_a.txt") - with open(file_a, "w", encoding='utf-8') as outfile: + with open(file_a, "w", encoding="utf-8") as outfile: outfile.write("hello world\n") repo.git.add(Git.polish_url(file_a)) repo.git.commit(message="Added file_a.txt") @@ -429,21 +429,21 @@ def test_rename_override(self, rw_dir): # create and commit file_b.txt with similarity index of 52 file_b = osp.join(rw_dir, "file_b.txt") - with open(file_b, "w", encoding='utf-8') as outfile: + with open(file_b, "w", encoding="utf-8") as outfile: outfile.write("hello world\nhello world") repo.git.add(Git.polish_url(file_b)) repo.git.commit(message="Removed file_a.txt. Added file_b.txt") - commit_a = repo.commit('HEAD') - commit_b = repo.commit('HEAD~1') + commit_a = repo.commit("HEAD") + commit_b = repo.commit("HEAD~1") # check default diff command with renamed files enabled diffs = commit_b.diff(commit_a) self.assertEqual(1, len(diffs)) diff = diffs[0] self.assertEqual(True, diff.renamed_file) - self.assertEqual('file_a.txt', diff.rename_from) - self.assertEqual('file_b.txt', diff.rename_to) + self.assertEqual("file_a.txt", diff.rename_from) + self.assertEqual("file_b.txt", diff.rename_to) # check diff with rename files disabled diffs = commit_b.diff(commit_a, no_renames=True) @@ -452,31 +452,31 @@ def test_rename_override(self, rw_dir): # check fileA.txt deleted diff = diffs[0] self.assertEqual(True, diff.deleted_file) - self.assertEqual('file_a.txt', diff.a_path) + self.assertEqual("file_a.txt", diff.a_path) # check fileB.txt added diff = diffs[1] self.assertEqual(True, diff.new_file) - self.assertEqual('file_b.txt', diff.a_path) + self.assertEqual("file_b.txt", diff.a_path) # check diff with high similarity index - diffs = commit_b.diff(commit_a, split_single_char_options=False, M='75%') + diffs = commit_b.diff(commit_a, split_single_char_options=False, M="75%") self.assertEqual(2, len(diffs)) # check fileA.txt deleted diff = diffs[0] self.assertEqual(True, diff.deleted_file) - self.assertEqual('file_a.txt', diff.a_path) + self.assertEqual("file_a.txt", diff.a_path) # check fileB.txt added diff = diffs[1] self.assertEqual(True, diff.new_file) - self.assertEqual('file_b.txt', diff.a_path) + self.assertEqual("file_b.txt", diff.a_path) # check diff with low similarity index - diffs = commit_b.diff(commit_a, split_single_char_options=False, M='40%') + diffs = commit_b.diff(commit_a, split_single_char_options=False, M="40%") self.assertEqual(1, len(diffs)) diff = diffs[0] self.assertEqual(True, diff.renamed_file) - self.assertEqual('file_a.txt', diff.rename_from) - self.assertEqual('file_b.txt', diff.rename_to) + self.assertEqual("file_a.txt", diff.rename_from) + self.assertEqual("file_b.txt", diff.rename_to) diff --git a/test/test_index.py b/test/test_index.py index 9b7ba52a6..fba9c78ec 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -946,7 +946,7 @@ def test_commit_msg_hook_fail(self, rw_repo): else: raise AssertionError("Should have caught a HookExecutionError") - @with_rw_repo('HEAD') + @with_rw_repo("HEAD") def test_index_add_pathlike(self, rw_repo): git_dir = Path(rw_repo.git_dir) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index eaee4e581..9dc7b8d2e 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -13,14 +13,13 @@ def tearDown(self): @with_rw_directory def test_init_repo_object(self, path_to_dir): - # [1-test_init_repo_object] # $ git init from git import Repo repo = Repo.init(path_to_dir) - # ![1-test_init_repo_object] + # ![1-test_init_repo_object] # [2-test_init_repo_object] repo = Repo(path_to_dir) @@ -28,9 +27,9 @@ def test_init_repo_object(self, path_to_dir): @with_rw_directory def test_cloned_repo_object(self, local_dir): - from git import Repo import git + # code to clone from url # [1-test_cloned_repo_object] # $ git clone @@ -44,9 +43,9 @@ def test_cloned_repo_object(self, local_dir): # [2-test_cloned_repo_object] # We must make a change to a file so that we can add the update to git - update_file = 'dir1/file2.txt' # we'll use local_dir/dir1/file2.txt - with open(f"{local_dir}/{update_file}", 'a') as f: - f.write('\nUpdate version 2') + update_file = "dir1/file2.txt" # we'll use local_dir/dir1/file2.txt + with open(f"{local_dir}/{update_file}", "a") as f: + f.write("\nUpdate version 2") # ![2-test_cloned_repo_object] # [3-test_cloned_repo_object] @@ -82,7 +81,7 @@ def test_cloned_repo_object(self, local_dir): # Untracked files - create new file # [7-test_cloned_repo_object] - f = open(f'{local_dir}/untracked.txt', 'w') # creates an empty file + f = open(f"{local_dir}/untracked.txt", "w") # creates an empty file f.close() # ![7-test_cloned_repo_object] @@ -95,8 +94,8 @@ def test_cloned_repo_object(self, local_dir): # [9-test_cloned_repo_object] # Let's modify one of our tracked files - with open(f'{local_dir}/Downloads/file3.txt', 'w') as f: - f.write('file3 version 2') # overwrite file 3 + with open(f"{local_dir}/Downloads/file3.txt", "w") as f: + f.write("file3 version 2") # overwrite file 3 # ![9-test_cloned_repo_object] # [10-test_cloned_repo_object] @@ -126,7 +125,7 @@ def test_cloned_repo_object(self, local_dir): # ![11.1-test_cloned_repo_object] # [11.2-test_cloned_repo_object] # lets add untracked.txt - repo.index.add(['untracked.txt']) + repo.index.add(["untracked.txt"]) diffs = repo.index.diff(repo.head.commit) for d in diffs: print(d.a_path) @@ -146,9 +145,7 @@ def test_cloned_repo_object(self, local_dir): # dir1/file2.txt # ![11.3-test_cloned_repo_object] - - - '''Trees and Blobs''' + """Trees and Blobs""" # Latest commit tree # [12-test_cloned_repo_object] @@ -195,7 +192,7 @@ def print_files_from_git(root, level=0): # Printing text files # [17-test_cloned_repo_object] - print_file = 'dir1/file2.txt' + print_file = "dir1/file2.txt" tree[print_file] # the head commit tree # Output @@ -221,4 +218,4 @@ def print_files_from_git(root, level=0): # Output # file 2 version 1 - # ![18.1-test_cloned_repo_object] \ No newline at end of file + # ![18.1-test_cloned_repo_object] diff --git a/test/test_repo.py b/test/test_repo.py index 1b46fba7c..6432b8c6f 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -251,7 +251,9 @@ def test_clone_from_with_path_contains_unicode(self): self.fail("Raised UnicodeEncodeError") @with_rw_directory - @skip("the referenced repository was removed, and one needs to setup a new password controlled repo under the orgs control") + @skip( + "the referenced repository was removed, and one needs to setup a new password controlled repo under the orgs control" + ) def test_leaking_password_in_clone_logs(self, rw_dir): password = "fakepassword1234" try: @@ -391,7 +393,9 @@ def test_clone_from_unsafe_options_allowed(self, rw_repo): for i, unsafe_option in enumerate(unsafe_options): destination = tmp_dir / str(i) assert not destination.exists() - Repo.clone_from(rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True) + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) assert destination.exists() @with_rw_repo("HEAD") @@ -755,8 +759,8 @@ def test_blame_complex_revision(self, git): @mock.patch.object(Git, "_call_process") def test_blame_accepts_rev_opts(self, git): res = self.rorepo.blame("HEAD", "README.md", rev_opts=["-M", "-C", "-C"]) - expected_args = ['blame', 'HEAD', '-M', '-C', '-C', '--', 'README.md'] - boilerplate_kwargs = {"p" : True, "stdout_as_string": False} + expected_args = ["blame", "HEAD", "-M", "-C", "-C", "--", "README.md"] + boilerplate_kwargs = {"p": True, "stdout_as_string": False} git.assert_called_once_with(*expected_args, **boilerplate_kwargs) @skipIf( @@ -1415,14 +1419,16 @@ def test_ignored_items_reported(self): gi = tmp_dir / "repo" / ".gitignore" - with open(gi, 'w') as file: - file.write('ignored_file.txt\n') - file.write('ignored_dir/\n') + with open(gi, "w") as file: + file.write("ignored_file.txt\n") + file.write("ignored_dir/\n") - assert temp_repo.ignored(['included_file.txt', 'included_dir/file.txt']) == [] - assert temp_repo.ignored(['ignored_file.txt']) == ['ignored_file.txt'] - assert temp_repo.ignored(['included_file.txt', 'ignored_file.txt']) == ['ignored_file.txt'] - assert temp_repo.ignored(['included_file.txt', 'ignored_file.txt', 'included_dir/file.txt', 'ignored_dir/file.txt']) == ['ignored_file.txt', 'ignored_dir/file.txt'] + assert temp_repo.ignored(["included_file.txt", "included_dir/file.txt"]) == [] + assert temp_repo.ignored(["ignored_file.txt"]) == ["ignored_file.txt"] + assert temp_repo.ignored(["included_file.txt", "ignored_file.txt"]) == ["ignored_file.txt"] + assert temp_repo.ignored( + ["included_file.txt", "ignored_file.txt", "included_dir/file.txt", "ignored_dir/file.txt"] + ) == ["ignored_file.txt", "ignored_dir/file.txt"] def test_ignored_raises_error_w_symlink(self): with tempfile.TemporaryDirectory() as tdir: diff --git a/test/test_submodule.py b/test/test_submodule.py index f7195626f..88717e220 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -39,11 +39,14 @@ def _patch_git_config(name, value): # This is recomputed each time the context is entered, for compatibility with # existing GIT_CONFIG_* environment variables, even if changed in this process. - patcher = mock.patch.dict(os.environ, { - "GIT_CONFIG_COUNT": str(pair_index + 1), - f"GIT_CONFIG_KEY_{pair_index}": name, - f"GIT_CONFIG_VALUE_{pair_index}": value, - }) + patcher = mock.patch.dict( + os.environ, + { + "GIT_CONFIG_COUNT": str(pair_index + 1), + f"GIT_CONFIG_KEY_{pair_index}": name, + f"GIT_CONFIG_VALUE_{pair_index}": value, + }, + ) with patcher: yield @@ -914,17 +917,17 @@ def test_ignore_non_submodule_file(self, rwdir): os.mkdir(smp) with open(osp.join(smp, "a"), "w", encoding="utf-8") as f: - f.write('test\n') + f.write("test\n") with open(osp.join(rwdir, ".gitmodules"), "w", encoding="utf-8") as f: - f.write("[submodule \"a\"]\n") + f.write('[submodule "a"]\n') f.write(" path = module\n") f.write(" url = https://github.com/chaconinc/DbConnector\n") parent.git.add(Git.polish_url(osp.join(smp, "a"))) parent.git.add(Git.polish_url(osp.join(rwdir, ".gitmodules"))) - parent.git.commit(message='test') + parent.git.commit(message="test") assert len(parent.submodules) == 0 @@ -1200,7 +1203,12 @@ def test_submodule_add_unsafe_options_allowed(self, rw_repo): # The options will be allowed, but the command will fail. with self.assertRaises(GitCommandError): Submodule.add( - rw_repo, "new", "new", str(tmp_dir), clone_multi_options=[unsafe_option], allow_unsafe_options=True + rw_repo, + "new", + "new", + str(tmp_dir), + clone_multi_options=[unsafe_option], + allow_unsafe_options=True, ) assert not tmp_file.exists() @@ -1211,7 +1219,12 @@ def test_submodule_add_unsafe_options_allowed(self, rw_repo): for unsafe_option in unsafe_options: with self.assertRaises(GitCommandError): Submodule.add( - rw_repo, "new", "new", str(tmp_dir), clone_multi_options=[unsafe_option], allow_unsafe_options=True + rw_repo, + "new", + "new", + str(tmp_dir), + clone_multi_options=[unsafe_option], + allow_unsafe_options=True, ) @with_rw_repo("HEAD") From bf7af69306ed8f14b33528165473ca3591a76246 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Sep 2023 21:29:33 -0400 Subject: [PATCH 0634/1790] Upgrade flake8 in pre-commit and fix new warnings Upgrading flake8 from 6.0.0 to 6.1.0 causes its pycodestyle dependency to be upgraded from 2.10.* to 2.11.*, which is desirable because: - Spurious "E231 missing whitespace after ':'" warnings on 3.12 due to the lack of full compatibility with Python 3.12 are gone. - New warnings appear, at least one of which, "E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`", does identify something we can improve. This upgrades flake8 in pre-commit and fixes the new warnings. --- .pre-commit-config.yaml | 2 +- git/objects/submodule/base.py | 2 +- git/refs/log.py | 2 +- git/util.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 581cb69b2..888e62d91 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 0d20305c6..c7e7856f0 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1403,7 +1403,7 @@ def iter_items( # END handle critical error # Make sure we are looking at a submodule object - if type(sm) != git.objects.submodule.base.Submodule: + if type(sm) is not git.objects.submodule.base.Submodule: continue # fill in remaining info - saves time as it doesn't have to be parsed again diff --git a/git/refs/log.py b/git/refs/log.py index 1f86356a4..1c2a2c470 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -244,7 +244,7 @@ def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": for i in range(index + 1): line = fp.readline() if not line: - raise IndexError(f"Index file ended at line {i+1}, before given index was reached") + raise IndexError(f"Index file ended at line {i + 1}, before given index was reached") # END abort on eof # END handle runup diff --git a/git/util.py b/git/util.py index 638807a36..48901ba0c 100644 --- a/git/util.py +++ b/git/util.py @@ -1136,7 +1136,7 @@ class IterableClassWatcher(type): def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: for base in bases: - if type(base) == IterableClassWatcher: + if type(base) is IterableClassWatcher: warnings.warn( f"GitPython Iterable subclassed by {name}. " "Iterable is deprecated due to naming clash since v3.1.18" From c1ec9cbd2a995585e53c90d61ae30102aaaff2a5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Sep 2023 22:25:40 -0400 Subject: [PATCH 0635/1790] Update flake8 additional dependencies, fix warning This bumps the versions of the flake8 plugins specified with pinned versions as additional dependencies of flake8 for pre-commit. Doing so gains a warning about a call to warnings.warn with no stacklevel argument. This appears to be the uncommon case where the implifit effect of stacklevel=1 is intended, so I have made that explicit, which clarifies this intent and dismisses the warning. --- .pre-commit-config.yaml | 4 ++-- git/repo/base.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 888e62d91..4c92656e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,8 +5,8 @@ repos: - id: flake8 additional_dependencies: [ - flake8-bugbear==22.12.6, - flake8-comprehensions==3.10.1, + flake8-bugbear==23.9.16, + flake8-comprehensions==3.14.0, flake8-typing-imports==1.14.0, ] exclude: ^doc|^git/ext/|^test/ diff --git a/git/repo/base.py b/git/repo/base.py index fda3cdc88..bc1b8876d 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -206,7 +206,8 @@ def __init__( if expand_vars and re.search(self.re_envvars, epath): warnings.warn( "The use of environment variables in paths is deprecated" - + "\nfor security reasons and may be removed in the future!!" + + "\nfor security reasons and may be removed in the future!!", + stacklevel=1, ) epath = expand_path(epath, expand_vars) if epath is not None: From 48441a94189494789f751e75f9e1919b9b700b13 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Sep 2023 23:40:44 -0400 Subject: [PATCH 0636/1790] Lint test/ (not just git/), fix warnings and a bug This expands flake8 linting to include the test suite, and fixes the resulting warnings. Four code changes are especially notable: - Unit tests from which documentation is automatically generated contain a few occurrences of "# @NoEffect". These suppressions are extended to "# noqa: B015 # @NoEffect" so the corresponding suppression is also applied for flake8. This is significant because it actually changes what appears in the code examples in the generated documentation. However, since the "@NoEffect" annotation was considered acceptable, this may be okay too. The resulting examples did not become excessively long. - Expressions like "[c for c in commits_for_file_generator]" appear in some unit tests from which documentation is automatically generated. The simpler form "list(commits_for_file_generator)" can replace them. This changes the examples in the documentation, but for the better, since that form is simpler (and also a better practice in general, thus a better thing to show readers). So I made those substitutions. - In test_repo.TestRepo.test_git_work_tree_env, the code intended to unpatch environment variables after the test was ineffective, instead causing os.environ to refer to an ordinary dict object that does not affect environment variables when written to. This uses unittest.mock.patch.dict instead, so the variables are unpatched and subsequent writes to environment variables in the test process are effective. - In test_submodule.TestSubmodule._do_base_tests, the expression statement "csm.module().head.ref.tracking_branch() is not None" appeared to be intended as an assertion, in spite of having been annoated @NoEffect. This is in light of the context and because, if the goal were only to exercise the function call, the "is not None" part would be superfluous. So I made it an assertion. --- .flake8 | 2 +- .pre-commit-config.yaml | 2 +- test/test_commit.py | 14 +++++++------- test/test_diff.py | 1 - test/test_docs.py | 8 ++++---- test/test_quick_doc.py | 14 ++++++-------- test/test_remote.py | 2 +- test/test_repo.py | 16 +++++++--------- test/test_submodule.py | 9 +++++---- 9 files changed, 32 insertions(+), 36 deletions(-) diff --git a/.flake8 b/.flake8 index 08001ffac..ed5d036bf 100644 --- a/.flake8 +++ b/.flake8 @@ -26,7 +26,7 @@ ignore = E265,E266,E731,E704, D, RST, RST3 -exclude = .tox,.venv,build,dist,doc,git/ext/,test +exclude = .tox,.venv,build,dist,doc,git/ext/ rst-roles = # for flake8-RST-docstrings attr,class,func,meth,mod,obj,ref,term,var # used by sphinx diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c92656e4..5a34b8af0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: flake8-comprehensions==3.14.0, flake8-typing-imports==1.14.0, ] - exclude: ^doc|^git/ext/|^test/ + exclude: ^doc|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 diff --git a/test/test_commit.py b/test/test_commit.py index f6fb49d50..527aea334 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -277,7 +277,7 @@ def __init__(self, *args, **kwargs): super(Child, self).__init__(*args, **kwargs) child_commits = list(Child.iter_items(self.rorepo, "master", paths=("CHANGES", "AUTHORS"))) - assert type(child_commits[0]) == Child + assert type(child_commits[0]) is Child def test_iter_items(self): # pretty not allowed @@ -525,12 +525,12 @@ def test_trailers(self): # check that trailer stays empty for multiple msg combinations msgs = [ - f"Subject\n", - f"Subject\n\nBody with some\nText\n", - f"Subject\n\nBody with\nText\n\nContinuation but\n doesn't contain colon\n", - f"Subject\n\nBody with\nText\n\nContinuation but\n only contains one :\n", - f"Subject\n\nBody with\nText\n\nKey: Value\nLine without colon\n", - f"Subject\n\nBody with\nText\n\nLine without colon\nKey: Value\n", + "Subject\n", + "Subject\n\nBody with some\nText\n", + "Subject\n\nBody with\nText\n\nContinuation but\n doesn't contain colon\n", + "Subject\n\nBody with\nText\n\nContinuation but\n only contains one :\n", + "Subject\n\nBody with\nText\n\nKey: Value\nLine without colon\n", + "Subject\n\nBody with\nText\n\nLine without colon\nKey: Value\n", ] for msg in msgs: diff --git a/test/test_diff.py b/test/test_diff.py index 9c3888f03..5aa4408bf 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -7,7 +7,6 @@ import ddt import shutil import tempfile -import unittest from git import ( Repo, GitCommandError, diff --git a/test/test_docs.py b/test/test_docs.py index 4c23e9f81..d6b88855f 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -263,9 +263,9 @@ def test_references_and_objects(self, rw_dir): # [8-test_references_and_objects] hc = repo.head.commit hct = hc.tree - hc != hct # @NoEffect - hc != repo.tags[0] # @NoEffect - hc == repo.head.reference.commit # @NoEffect + hc != hct # noqa: B015 # @NoEffect + hc != repo.tags[0] # noqa: B015 # @NoEffect + hc == repo.head.reference.commit # noqa: B015 # @NoEffect # ![8-test_references_and_objects] # [9-test_references_and_objects] @@ -369,7 +369,7 @@ def test_references_and_objects(self, rw_dir): # The index contains all blobs in a flat list assert len(list(index.iter_blobs())) == len([o for o in repo.head.commit.tree.traverse() if o.type == "blob"]) # Access blob objects - for (_path, _stage), entry in index.entries.items(): + for (_path, _stage), _entry in index.entries.items(): pass new_file_path = os.path.join(repo.working_tree_dir, "new-file-name") open(new_file_path, "w").close() diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 9dc7b8d2e..342a7f293 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -1,6 +1,3 @@ -import pytest - - from test.lib import TestBase from test.lib.helper import with_rw_directory @@ -25,10 +22,11 @@ def test_init_repo_object(self, path_to_dir): repo = Repo(path_to_dir) # ![2-test_init_repo_object] + del repo # Avoids "assigned to but never used" warning. Doesn't go in the docs. + @with_rw_directory def test_cloned_repo_object(self, local_dir): from git import Repo - import git # code to clone from url # [1-test_cloned_repo_object] @@ -72,7 +70,7 @@ def test_cloned_repo_object(self, local_dir): # [6-test_cloned_repo_object] commits_for_file_generator = repo.iter_commits(all=True, max_count=10, paths=update_file) - commits_for_file = [c for c in commits_for_file_generator] + commits_for_file = list(commits_for_file_generator) commits_for_file # Outputs: [, @@ -136,7 +134,7 @@ def test_cloned_repo_object(self, local_dir): # Compare commit to commit # [11.3-test_cloned_repo_object] - first_commit = [c for c in repo.iter_commits(all=True)][-1] + first_commit = list(repo.iter_commits(all=True))[-1] diffs = repo.head.commit.diff(first_commit) for d in diffs: print(d.a_path) @@ -154,7 +152,7 @@ def test_cloned_repo_object(self, local_dir): # Previous commit tree # [13-test_cloned_repo_object] - prev_commits = [c for c in repo.iter_commits(all=True, max_count=10)] # last 10 commits from all branches + prev_commits = list(repo.iter_commits(all=True, max_count=10)) # last 10 commits from all branches tree = prev_commits[0].tree # ![13-test_cloned_repo_object] @@ -210,7 +208,7 @@ def print_files_from_git(root, level=0): # print previous tree # [18.1-test_cloned_repo_object] - commits_for_file = [c for c in repo.iter_commits(all=True, paths=print_file)] + commits_for_file = list(repo.iter_commits(all=True, paths=print_file)) tree = commits_for_file[-1].tree # gets the first commit tree blob = tree[print_file] diff --git a/test/test_remote.py b/test/test_remote.py index e0dcb4131..7144b2791 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -160,7 +160,7 @@ def _do_test_push_result(self, results, remote): # END error checking # END for each info - if any([info.flags & info.ERROR for info in results]): + if any(info.flags & info.ERROR for info in results): self.assertRaises(GitCommandError, results.raise_if_error) else: # No errors, so this should do nothing diff --git a/test/test_repo.py b/test/test_repo.py index 6432b8c6f..d3bc864cd 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -252,7 +252,8 @@ def test_clone_from_with_path_contains_unicode(self): @with_rw_directory @skip( - "the referenced repository was removed, and one needs to setup a new password controlled repo under the orgs control" + """The referenced repository was removed, and one needs to set up a new + password controlled repo under the org's control.""" ) def test_leaking_password_in_clone_logs(self, rw_dir): password = "fakepassword1234" @@ -758,9 +759,9 @@ def test_blame_complex_revision(self, git): @mock.patch.object(Git, "_call_process") def test_blame_accepts_rev_opts(self, git): - res = self.rorepo.blame("HEAD", "README.md", rev_opts=["-M", "-C", "-C"]) expected_args = ["blame", "HEAD", "-M", "-C", "-C", "--", "README.md"] boilerplate_kwargs = {"p": True, "stdout_as_string": False} + self.rorepo.blame("HEAD", "README.md", rev_opts=["-M", "-C", "-C"]) git.assert_called_once_with(*expected_args, **boilerplate_kwargs) @skipIf( @@ -971,7 +972,7 @@ def _assert_rev_parse(self, name): # history with number ni = 11 history = [obj.parents[0]] - for pn in range(ni): + for _ in range(ni): history.append(history[-1].parents[0]) # END get given amount of commits @@ -1329,6 +1330,7 @@ def test_git_work_tree_env(self, rw_dir): # move .git directory to a subdirectory # set GIT_DIR and GIT_WORK_TREE appropriately # check that repo.working_tree_dir == rw_dir + self.rorepo.clone(join_path_native(rw_dir, "master_repo")) repo_dir = join_path_native(rw_dir, "master_repo") @@ -1338,16 +1340,12 @@ def test_git_work_tree_env(self, rw_dir): os.mkdir(new_subdir) os.rename(old_git_dir, new_git_dir) - oldenv = os.environ.copy() - os.environ["GIT_DIR"] = new_git_dir - os.environ["GIT_WORK_TREE"] = repo_dir + to_patch = {"GIT_DIR": new_git_dir, "GIT_WORK_TREE": repo_dir} - try: + with mock.patch.dict(os.environ, to_patch): r = Repo() self.assertEqual(r.working_tree_dir, repo_dir) self.assertEqual(r.working_dir, repo_dir) - finally: - os.environ = oldenv @with_rw_directory def test_rebasing(self, rw_dir): diff --git a/test/test_submodule.py b/test/test_submodule.py index 88717e220..35ff0d7a8 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -111,7 +111,7 @@ def _do_base_tests(self, rwrepo): # force it to reread its information del smold._url - smold.url == sm.url # @NoEffect + smold.url == sm.url # noqa: B015 # @NoEffect # test config_reader/writer methods sm.config_reader() @@ -248,7 +248,7 @@ def _do_base_tests(self, rwrepo): assert csm.module_exists() # tracking branch once again - csm.module().head.ref.tracking_branch() is not None # @NoEffect + assert csm.module().head.ref.tracking_branch() is not None # this flushed in a sub-submodule assert len(list(rwrepo.iter_submodules())) == 2 @@ -480,8 +480,9 @@ def test_base_bare(self, rwrepo): File "C:\\projects\\gitpython\\git\\cmd.py", line 559, in execute raise GitCommandNotFound(command, err) git.exc.GitCommandNotFound: Cmd('git') not found due to: OSError('[WinError 6] The handle is invalid') - cmdline: git clone -n --shared -v C:\\projects\\gitpython\\.git Users\\appveyor\\AppData\\Local\\Temp\\1\\tmplyp6kr_rnon_bare_test_root_module""", - ) # noqa E501 + cmdline: git clone -n --shared -v C:\\projects\\gitpython\\.git Users\\appveyor\\AppData\\Local\\Temp\\1\\tmplyp6kr_rnon_bare_test_root_module + """, # noqa E501 + ) @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): # Can query everything without problems From 98877c58cd402ea94d4235ab9d4074db76dca74d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Sep 2023 02:46:48 -0400 Subject: [PATCH 0637/1790] Refactor "finally" cleanup in tests, fix minor bug This refactors code in the test suite that uses try-finally, to simplify and clarify what it is doing. An exception to this being a refactoring is that a possible bug is fixed where a throwaway environment variable "FOO" was patched for a test and never unpatched. There are two patterns refactored here: - try-(try-except)-finally to try-except-finally. When the entire suite of the try-block in try-finally is itself a try-except, that has the same effect as try-except-finally. (Python always attempts to run the finally suite in either case, and does so after any applicable except suite is run.) - Replacing try-finally with an appropriate context manager. (These changes do not fix, nor introduce, any flake8 errors, but they were found by manual search for possible problems related to recent flake8 findings but outside of what flake8 can catch. To limit scope, this only refactors try-finally in the test suite.) --- test/lib/helper.py | 32 +++++++++++++++----------------- test/test_git.py | 17 ++++++----------- test/test_repo.py | 9 ++------- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 64c70a8f4..e8464b7d4 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -94,17 +94,16 @@ def wrapper(self): os.mkdir(path) keep = False try: - try: - return func(self, path) - except Exception: - log.info( - "Test %s.%s failed, output is at %r\n", - type(self).__name__, - func.__name__, - path, - ) - keep = True - raise + return func(self, path) + except Exception: + log.info( + "Test %s.%s failed, output is at %r\n", + type(self).__name__, + func.__name__, + path, + ) + keep = True + raise finally: # Need to collect here to be sure all handles have been closed. It appears # a windows-only issue. In fact things should be deleted, as well as @@ -147,12 +146,11 @@ def repo_creator(self): prev_cwd = os.getcwd() os.chdir(rw_repo.working_dir) try: - try: - return func(self, rw_repo) - except: # noqa E722 - log.info("Keeping repo after failure: %s", repo_dir) - repo_dir = None - raise + return func(self, rw_repo) + except: # noqa E722 + log.info("Keeping repo after failure: %s", repo_dir) + repo_dir = None + raise finally: os.chdir(prev_cwd) rw_repo.git.clear_cache() diff --git a/test/test_git.py b/test/test_git.py index 2c392155a..31065d8fd 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -195,17 +195,12 @@ def test_version(self): # END verify number types def test_cmd_override(self): - prev_cmd = self.git.GIT_PYTHON_GIT_EXECUTABLE - exc = GitCommandNotFound - try: - # set it to something that doesn't exist, assure it raises - type(self.git).GIT_PYTHON_GIT_EXECUTABLE = osp.join( - "some", "path", "which", "doesn't", "exist", "gitbinary" - ) - self.assertRaises(exc, self.git.version) - finally: - type(self.git).GIT_PYTHON_GIT_EXECUTABLE = prev_cmd - # END undo adjustment + with mock.patch.object( + type(self.git), + "GIT_PYTHON_GIT_EXECUTABLE", + osp.join("some", "path", "which", "doesn't", "exist", "gitbinary"), + ): + self.assertRaises(GitCommandNotFound, self.git.version) def test_refresh(self): # test a bad git path refresh diff --git a/test/test_repo.py b/test/test_repo.py index d3bc864cd..15899ec50 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -847,18 +847,13 @@ def test_comparison_and_hash(self): @with_rw_directory def test_tilde_and_env_vars_in_repo_path(self, rw_dir): - ph = os.environ.get("HOME") - try: + with mock.patch.dict(os.environ, {"HOME": rw_dir}): os.environ["HOME"] = rw_dir Repo.init(osp.join("~", "test.git"), bare=True) + with mock.patch.dict(os.environ, {"FOO": rw_dir}): os.environ["FOO"] = rw_dir Repo.init(osp.join("$FOO", "test.git"), bare=True) - finally: - if ph: - os.environ["HOME"] = ph - del os.environ["FOO"] - # end assure HOME gets reset to what it was def test_git_cmd(self): # test CatFileContentStream, just to be very sure we have no fencepost errors From 46d3d0520d8877b1979e6385bada9d9e1e0731ec Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Thu, 21 Sep 2023 12:00:59 +0200 Subject: [PATCH 0638/1790] Add more checks for the validity of refnames This change adds checks based on the rules described in [0] in order to more robustly check a refname's validity. [0]: https://git-scm.com/docs/git-check-ref-format --- git/refs/symbolic.py | 50 ++++++++++++++++++++++++++++++++++++++++++-- test/test_refs.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 5c293aa7b..819615103 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -161,6 +161,51 @@ def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> return hexsha # END recursive dereferencing + @staticmethod + def _check_ref_name_valid(ref_path: PathLike) -> None: + # Based on the rules described in https://git-scm.com/docs/git-check-ref-format/#_description + previous: Union[str, None] = None + one_before_previous: Union[str, None] = None + for c in str(ref_path): + if c in " ~^:?*[\\": + raise ValueError( + f"Invalid reference '{ref_path}': references cannot contain spaces, tildes (~), carets (^)," + f" colons (:), question marks (?), asterisks (*), open brackets ([) or backslashes (\\)" + ) + elif c == ".": + if previous is None or previous == "/": + raise ValueError( + f"Invalid reference '{ref_path}': references cannot start with a period (.) or contain '/.'" + ) + elif previous == ".": + raise ValueError(f"Invalid reference '{ref_path}': references cannot contain '..'") + elif c == "/": + if previous == "/": + raise ValueError(f"Invalid reference '{ref_path}': references cannot contain '//'") + elif previous is None: + raise ValueError( + f"Invalid reference '{ref_path}': references cannot start with forward slashes '/'" + ) + elif c == "{" and previous == "@": + raise ValueError(f"Invalid reference '{ref_path}': references cannot contain '@{{'") + elif ord(c) < 32 or ord(c) == 127: + raise ValueError(f"Invalid reference '{ref_path}': references cannot contain ASCII control characters") + + one_before_previous = previous + previous = c + + if previous == ".": + raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a period (.)") + elif previous == "/": + raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a forward slash (/)") + elif previous == "@" and one_before_previous is None: + raise ValueError(f"Invalid reference '{ref_path}': references cannot be '@'") + elif any([component.endswith(".lock") for component in str(ref_path).split("/")]): + raise ValueError( + f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" + f" '.lock'" + ) + @classmethod def _get_ref_info_helper( cls, repo: "Repo", ref_path: Union[PathLike, None] @@ -168,8 +213,9 @@ def _get_ref_info_helper( """Return: (str(sha), str(target_ref_path)) if available, the sha the file at rela_path points to, or None. target_ref_path is the reference we point to, or None""" - if ".." in str(ref_path): - raise ValueError(f"Invalid reference '{ref_path}'") + if ref_path: + cls._check_ref_name_valid(ref_path) + tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: diff --git a/test/test_refs.py b/test/test_refs.py index afd273df9..80166f651 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -631,3 +631,39 @@ def test_refs_outside_repo(self): ref_file.flush() ref_file_name = Path(ref_file.name).name self.assertRaises(BadName, self.rorepo.commit, f"../../{ref_file_name}") + + def test_validity_ref_names(self): + check_ref = SymbolicReference._check_ref_name_valid + # Based on the rules specified in https://git-scm.com/docs/git-check-ref-format/#_description + # Rule 1 + self.assertRaises(ValueError, check_ref, ".ref/begins/with/dot") + self.assertRaises(ValueError, check_ref, "ref/component/.begins/with/dot") + self.assertRaises(ValueError, check_ref, "ref/ends/with/a.lock") + self.assertRaises(ValueError, check_ref, "ref/component/ends.lock/with/period_lock") + # Rule 2 + check_ref("valid_one_level_refname") + # Rule 3 + self.assertRaises(ValueError, check_ref, "ref/contains/../double/period") + # Rule 4 + for c in " ~^:": + self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{c}/character") + for code in range(0, 32): + self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{chr(code)}/ASCII/control_character") + self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{chr(127)}/ASCII/control_character") + # Rule 5 + for c in "*?[": + self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{c}/character") + # Rule 6 + self.assertRaises(ValueError, check_ref, "/ref/begins/with/slash") + self.assertRaises(ValueError, check_ref, "ref/ends/with/slash/") + self.assertRaises(ValueError, check_ref, "ref/contains//double/slash/") + # Rule 7 + self.assertRaises(ValueError, check_ref, "ref/ends/with/dot.") + # Rule 8 + self.assertRaises(ValueError, check_ref, "ref/contains@{/at_brace") + # Rule 9 + self.assertRaises(ValueError, check_ref, "@") + # Rule 10 + self.assertRaises(ValueError, check_ref, "ref/contain\\s/backslash") + # Valid reference name should not raise + check_ref("valid/ref/name") From c5693204f772102f289c2e4269a399370a38d82b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Sep 2023 06:40:03 -0400 Subject: [PATCH 0639/1790] Make an old mock.patch.dict on os.environ clearer This clarifies that the object's contents are patched, rather than patching the attribute used to get the object in the first place. It is also in keeping with the style of patching os.environ elsewhere in the test suite. --- test/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_git.py b/test/test_git.py index 31065d8fd..481309538 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -245,7 +245,7 @@ def test_insert_after_kwarg_raises(self): def test_env_vars_passed_to_git(self): editor = "non_existent_editor" - with mock.patch.dict("os.environ", {"GIT_EDITOR": editor}): # @UndefinedVariable + with mock.patch.dict(os.environ, {"GIT_EDITOR": editor}): self.assertEqual(self.git.var("GIT_EDITOR"), editor) @with_rw_directory From 592ec8492326ccef7b4af022bffcc385eac8adb8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Sep 2023 06:57:27 -0400 Subject: [PATCH 0640/1790] Fix rollback bug in SymbolicReference.set_reference This fixes the initialization of the "ok" flag by setting it to False before the operation that might fail is attempted. (This is a fix for the *specific* problem reported in #1669 about how the pattern SymbolicReference.set_reference attempts to use with LockedFD is not correctly implemented.) --- git/refs/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 5c293aa7b..6361713de 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -370,7 +370,7 @@ def set_reference( lfd = LockedFD(fpath) fd = lfd.open(write=True, stream=True) - ok = True + ok = False try: fd.write(write_value.encode("utf-8") + b"\n") lfd.commit() From ff84b26445b147ee9e2c75d82903b0c6b09e2b7a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Sep 2023 03:00:04 -0400 Subject: [PATCH 0641/1790] Refactor try-finally cleanup in git/ This is, in part, to help avoid (or be able to notice) other bugs like the rollback bug that affected SymbolicReference. However, it also includes a simplification of try-(try-except)-finally to try-except-finally. --- git/config.py | 17 ++++++++--------- git/index/base.py | 8 +++----- git/refs/symbolic.py | 8 +++----- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/git/config.py b/git/config.py index 880eb9301..76b149179 100644 --- a/git/config.py +++ b/git/config.py @@ -406,15 +406,14 @@ def release(self) -> None: return try: - try: - self.write() - except IOError: - log.error("Exception during destruction of GitConfigParser", exc_info=True) - except ReferenceError: - # This happens in PY3 ... and usually means that some state cannot be written - # as the sections dict cannot be iterated - # Usually when shutting down the interpreter, don'y know how to fix this - pass + self.write() + except IOError: + log.error("Exception during destruction of GitConfigParser", exc_info=True) + except ReferenceError: + # This happens in PY3 ... and usually means that some state cannot be + # written as the sections dict cannot be iterated + # Usually when shutting down the interpreter, don't know how to fix this + pass finally: if self._lock is not None: self._lock._release_lock() diff --git a/git/index/base.py b/git/index/base.py index cf016df6a..0cdeb1ce5 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -224,13 +224,11 @@ def write( lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) - ok = False try: self._serialize(stream, ignore_extension_data) - ok = True - finally: - if not ok: - lfd.rollback() + except BaseException: + lfd.rollback() + raise lfd.commit() diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 6361713de..734bf32d8 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -370,14 +370,12 @@ def set_reference( lfd = LockedFD(fpath) fd = lfd.open(write=True, stream=True) - ok = False try: fd.write(write_value.encode("utf-8") + b"\n") lfd.commit() - ok = True - finally: - if not ok: - lfd.rollback() + except BaseException: + lfd.rollback() + raise # Adjust the reflog if logmsg is not None: self.log_append(oldbinsha, logmsg) From e480985aa4d358d0cc167d4552910e85944b8966 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Sep 2023 07:05:40 -0400 Subject: [PATCH 0642/1790] Tweak rollback logic in log.to_file This modifies the exception handling in log.to_file so it catches BaseException rather than Exception and rolls back. Ordinarily we do not want to catch BaseException, since this means catching things like SystemExit, KeyboardInterupt, etc., but the other cases of rolling back with LockedFD do it that strongly (both before when try-finally was used with a flag, and now with try-except catching BaseException to roll back the temporary-file write and reraise). Having this behave subtly different does not appear intentional. (This is also closer to what will happen if LockedFD becomes a context manager and these pieces of code use it in a with-statement: even exceptions not inheriting from Exception will cause __exit__ to be called.) --- git/refs/log.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/git/refs/log.py b/git/refs/log.py index 1f86356a4..9acc0e360 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -262,8 +262,7 @@ def to_file(self, filepath: PathLike) -> None: try: self._serialize(fp) lfd.commit() - except Exception: - # on failure it rolls back automatically, but we make it clear + except BaseException: lfd.rollback() raise # END handle change From a4701a0f17308ec8d4b5871e6e2a95c4e2ca5b91 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Sep 2023 19:12:03 -0400 Subject: [PATCH 0643/1790] Remove `@NoEffect` annotations + Add missing asserts, where an expression statement was by itself that was intended as an assertion. This turned out to be the case in the places `@NoEffect` appeared in rendered documentation, making it so no per-file-ignores or other broadened suppressions were needed. + Fix misspellings (including one case affecting documentation). + Add a FIXME comment for investigating a free-standing expression statement with no obvious side effects that may have been meant as an assertion but would fail if turned into one. --- test/test_docs.py | 10 +++++----- test/test_refs.py | 4 ++-- test/test_submodule.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_docs.py b/test/test_docs.py index d6b88855f..79e1f1be4 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -167,7 +167,7 @@ def update(self, op_code, cur_count, max_count=None, message=""): open(new_file_path, "wb").close() # create new file in working tree cloned_repo.index.add([new_file_path]) # add it to the index # Commit the changes to deviate masters history - cloned_repo.index.commit("Added a new file in the past - for later merege") + cloned_repo.index.commit("Added a new file in the past - for later merge") # prepare a merge master = cloned_repo.heads.master # right-hand side is ahead of us, in the future @@ -198,7 +198,7 @@ def update(self, op_code, cur_count, max_count=None, message=""): # .gitmodules was written and added to the index, which is now being committed cloned_repo.index.commit("Added submodule") - assert sm.exists() and sm.module_exists() # this submodule is defintely available + assert sm.exists() and sm.module_exists() # this submodule is definitely available sm.remove(module=True, configuration=False) # remove the working tree assert sm.exists() and not sm.module_exists() # the submodule itself is still available @@ -263,9 +263,9 @@ def test_references_and_objects(self, rw_dir): # [8-test_references_and_objects] hc = repo.head.commit hct = hc.tree - hc != hct # noqa: B015 # @NoEffect - hc != repo.tags[0] # noqa: B015 # @NoEffect - hc == repo.head.reference.commit # noqa: B015 # @NoEffect + assert hc != hct + assert hc != repo.tags[0] + assert hc == repo.head.reference.commit # ![8-test_references_and_objects] # [9-test_references_and_objects] diff --git a/test/test_refs.py b/test/test_refs.py index afd273df9..7598deb08 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -386,7 +386,7 @@ def test_head_reset(self, rw_repo): head_tree = head.commit.tree self.assertRaises(ValueError, setattr, head, "commit", head_tree) assert head.commit == old_commit # and the ref did not change - # we allow heds to point to any object + # we allow heads to point to any object head.object = head_tree assert head.object == head_tree # cannot query tree as commit @@ -489,7 +489,7 @@ def test_head_reset(self, rw_repo): cur_head.reference.commit, ) # it works if the new ref points to the same reference - assert SymbolicReference.create(rw_repo, symref.path, symref.reference).path == symref.path # @NoEffect + assert SymbolicReference.create(rw_repo, symref.path, symref.reference).path == symref.path SymbolicReference.delete(rw_repo, symref) # would raise if the symref wouldn't have been deletedpbl symref = SymbolicReference.create(rw_repo, symref_path, cur_head.reference) diff --git a/test/test_submodule.py b/test/test_submodule.py index 35ff0d7a8..4a9c9c582 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -111,7 +111,7 @@ def _do_base_tests(self, rwrepo): # force it to reread its information del smold._url - smold.url == sm.url # noqa: B015 # @NoEffect + smold.url == sm.url # noqa: B015 # FIXME: Should this be an assertion? # test config_reader/writer methods sm.config_reader() From 832b6eeb4a14e669099c486862c9f568215d5afb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 22 Sep 2023 09:04:21 +0200 Subject: [PATCH 0644/1790] remove unnecessary list comprehension to fix CI --- git/refs/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 4d087e7a7..549160444 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -200,7 +200,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a forward slash (/)") elif previous == "@" and one_before_previous is None: raise ValueError(f"Invalid reference '{ref_path}': references cannot be '@'") - elif any([component.endswith(".lock") for component in str(ref_path).split("/")]): + elif any(component.endswith(".lock") for component in str(ref_path).split("/")): raise ValueError( f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" f" '.lock'" From 0bd2890ef42a7506b81a96c3c94b064917ed0d7b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 22 Sep 2023 09:33:54 +0200 Subject: [PATCH 0645/1790] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b402c1a8b..1f1a39706 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.36 +3.1.37 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 06ec4b72c..a789b068d 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,15 @@ Changelog ========= +3.1.37 +====== + +This release contains another security fix that further improves validation of symbolic references +and thus properly fixes this CVE: https://github.com/advisories/GHSA-cwvm-v4w8-q58c . + +See the following for all changes. +https://github.com/gitpython-developers/gitpython/milestone/67?closed=1 + 3.1.36 ====== From b27a89f683cda85ebd78243c055e876282df89ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 22 Sep 2023 09:37:09 +0200 Subject: [PATCH 0646/1790] fix makefile to compare commit hashes only Otherwise, with signed commits, the latest-tag-HEAD comparison would always fail. --- check-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-version.sh b/check-version.sh index 6d10d6785..c50bf498b 100755 --- a/check-version.sh +++ b/check-version.sh @@ -29,7 +29,7 @@ changes_version="$(awk '/^[0-9]/ {print $0; exit}' "$changes_path")" config_opts="$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" latest_tag="$(git $config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1)" head_sha="$(git rev-parse HEAD)" -latest_tag_sha="$(git rev-parse "$latest_tag")" +latest_tag_sha="$(git rev-parse "${latest_tag}^{commit}")" # Display a table of all the current version, tag, and HEAD commit information. echo $'\nThe VERSION must be the same in all locations, and so must the HEAD and tag SHA' From 4e701bdae829bb02abd7f9acf5e6508e242b6977 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 05:54:47 -0400 Subject: [PATCH 0647/1790] Add missing assert keywords This turns two tuple expression statements--each of an equality comparison and message--that were not being checked or otherwise used, and that were intended to be assertions, into assertions. --- test/test_submodule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 4a9c9c582..0aa80e5ce 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1031,8 +1031,8 @@ def test_branch_renames(self, rw_dir): # This doesn't fail as our own submodule binsha didn't change, and the reset is only triggered if # to latest revision is True. parent_repo.submodule_update(to_latest_revision=False) - sm_mod.head.ref.name == sm_pfb.name, "should have been switched to past head" - sm_mod.commit() == sm_fb.commit, "Head wasn't reset" + assert sm_mod.head.ref.name == sm_pfb.name, "should have been switched to past head" + assert sm_mod.commit() == sm_fb.commit, "Head wasn't reset" self.assertRaises(RepositoryDirtyError, parent_repo.submodule_update, to_latest_revision=True) parent_repo.submodule_update(to_latest_revision=True, force_reset=True) From 45773c27fa40fddf72b40970836b3649094cd994 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Sep 2023 08:16:29 -0400 Subject: [PATCH 0648/1790] Instrument workflows to investigate skipped tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes jobs from both test workflows give more information relevant to examining which tests are skipped (and if any tests xfail, those too) in what environments: - Values of os.name and git.util.is_win. - The name of each test that runs, with its status. The latter doesn't increase the output length as much as might be expected, because due to the way the output is handled, the pytest-sugar pretty output format without -v looked like: test/test_actor.py ✓ 0% test/test_actor.py ✓✓ 0% ▏ test/test_actor.py ✓✓✓ 1% ▏ test/test_actor.py ✓✓✓✓ 1% ▏ When instead it was intended to fit on a single line. Still, the current output with -v has extra newlines, increasing length and worsening readability, so it should be improved on if possible. --- .github/workflows/cygwin-test.yml | 6 +++++- .github/workflows/pythonpackage.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 962791ae7..3f93f6830 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -58,7 +58,11 @@ jobs: run: | /usr/bin/python -m pip install ".[test]" + - name: Check 'is_win' + run: | + /usr/bin/python -c 'import os, git; print(f"os.name={os.name}, is_win={git.compat.is_win}")' + - name: Test with pytest run: | set +x - /usr/bin/python -m pytest + /usr/bin/python -m pytest -v diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a5467ef94..c57f31cdc 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -73,9 +73,13 @@ jobs: # so we have to ignore errors until that changes. continue-on-error: true + - name: Check 'is_win' + run: | + python -c 'import os, git; print(f"os.name={os.name}, is_win={git.compat.is_win}")' + - name: Test with pytest run: | - pytest + pytest -v continue-on-error: false - name: Documentation From 6fbe5118514618d34ea8912e99fbde3fd6d7a557 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 16:09:01 -0400 Subject: [PATCH 0649/1790] Show version and platform info in one place Instead of splitting it in into two places where at least one of the places is highly likely to be missed, this puts it together just before the first steps that makes nontrivial use of the installed packages. Grouping it together, it can't really be shown earlier, because one of the pieces of information is obtained using the git module (to examine that behavior of the code). This also presents the information more clearly. "set -x" makes this easy, so the commands are rewritten to take advantage of it. --- .github/workflows/cygwin-test.yml | 13 ++++++------- .github/workflows/pythonpackage.yml | 17 ++++++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 3f93f6830..1563afc95 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -29,11 +29,6 @@ jobs: with: packages: python39 python39-pip python39-virtualenv git - - name: Show python and git versions - run: | - /usr/bin/python --version - /usr/bin/git version - - name: Tell git to trust this repo run: | /usr/bin/git config --global --add safe.directory "$(pwd)" @@ -58,9 +53,13 @@ jobs: run: | /usr/bin/python -m pip install ".[test]" - - name: Check 'is_win' + - name: Show version and platform information run: | - /usr/bin/python -c 'import os, git; print(f"os.name={os.name}, is_win={git.compat.is_win}")' + /usr/bin/git version + /usr/bin/python --version + /usr/bin/python -c 'import sys; print(sys.platform)' + /usr/bin/python -c 'import os; print(os.name)' + /usr/bin/python -c 'import git; print(git.compat.is_win)' - name: Test with pytest run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c57f31cdc..696057ae2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -36,11 +36,6 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} - - name: Show python and git versions - run: | - python --version - git version - - name: Prepare this repo for tests run: | TRAVIS=yes ./init-tests-after-clone.sh @@ -66,6 +61,14 @@ jobs: run: | pip install ".[test]" + - name: Show version and platform information + run: | + git version + python --version + python -c 'import sys; print(sys.platform)' + python -c 'import os; print(os.name)' + python -c 'import git; print(git.compat.is_win)' + - name: Check types with mypy run: | mypy -p git @@ -73,10 +76,6 @@ jobs: # so we have to ignore errors until that changes. continue-on-error: true - - name: Check 'is_win' - run: | - python -c 'import os, git; print(f"os.name={os.name}, is_win={git.compat.is_win}")' - - name: Test with pytest run: | pytest -v From bd3307ad428025e72fb66a9ca4e8231aa0917f9e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 16:29:43 -0400 Subject: [PATCH 0650/1790] Make "Update PyPA packages" step clearer --- .github/workflows/pythonpackage.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 696057ae2..a311798e2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -50,12 +50,12 @@ jobs: - name: Update PyPA packages run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel + + # Python prior to 3.12 ships setuptools. Upgrade it if present. if pip freeze --all | grep --quiet '^setuptools=='; then - # Python prior to 3.12 ships setuptools. Upgrade it if present. python -m pip install --upgrade setuptools fi - python -m pip install --upgrade wheel - name: Install project and test dependencies run: | From 680d7957373c6bf193388907e3dbb770f3867ffe Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Sep 2023 17:13:01 -0400 Subject: [PATCH 0651/1790] Show all the failures Don't stop after the first 10. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fa06458eb..0466ed4c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] python_files = 'test_*.py' testpaths = 'test' # space separated list of paths from root e.g test tests doc/testing -addopts = '--cov=git --cov-report=term --maxfail=10 --force-sugar --disable-warnings' +addopts = '--cov=git --cov-report=term --force-sugar --disable-warnings' filterwarnings = 'ignore::DeprecationWarning' # --cov coverage # --cov-report term # send report to terminal term-missing -> terminal with line numbers html xml From 75cf5402d0d6c6712c1b0f5bd114cc9fd8780edc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 01:39:33 -0400 Subject: [PATCH 0652/1790] Keep sugar for local use, but use instafail on CI There are two benefits of the pytest-sugar plugin: 1. Pretty output. 2. Show details on each failure immediately instead of at the end. The first benefit is effectively local-only, because extra newlines are appearing when it runs on CI, both with and without -v. The second benefit applies both locally and on CI. So this adds the pytest-instafail plugin and uses it on CI to get the second benefit. It is not set up to run automatically, and pytest-sugar still is (though no longer forced), so local testing retains no benefit and we don't have a clash. The name "instafail" refers only to instantly *seeing* failures: it does not cause the pytest runner to stop earlier than otherwise. --- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- pyproject.toml | 2 +- test-requirements.txt | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 1563afc95..cae828099 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -64,4 +64,4 @@ jobs: - name: Test with pytest run: | set +x - /usr/bin/python -m pytest -v + /usr/bin/python -m pytest -p no:sugar -v --instafail diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a311798e2..9ac1088f7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -78,7 +78,7 @@ jobs: - name: Test with pytest run: | - pytest -v + pytest -v -p no:sugar --instafail continue-on-error: false - name: Documentation diff --git a/pyproject.toml b/pyproject.toml index 0466ed4c4..f4fc33fec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] python_files = 'test_*.py' testpaths = 'test' # space separated list of paths from root e.g test tests doc/testing -addopts = '--cov=git --cov-report=term --force-sugar --disable-warnings' +addopts = '--cov=git --cov-report=term --disable-warnings' filterwarnings = 'ignore::DeprecationWarning' # --cov coverage # --cov-report term # send report to terminal term-missing -> terminal with line numbers html xml diff --git a/test-requirements.txt b/test-requirements.txt index b00dd6f06..1c08c736f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,4 +5,5 @@ mypy pre-commit pytest pytest-cov +pytest-instafail pytest-sugar From eb56e7bdf15344739d3c2d671c1ca7dc185b8abe Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 02:01:25 -0400 Subject: [PATCH 0653/1790] Pass -v twice to see full skip reasons + Reorder pytest arguments so both workflows are consistent. --- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index cae828099..8d1145e76 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -64,4 +64,4 @@ jobs: - name: Test with pytest run: | set +x - /usr/bin/python -m pytest -p no:sugar -v --instafail + /usr/bin/python -m pytest -p no:sugar --instafail -vv diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9ac1088f7..c0402e4bb 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -78,7 +78,7 @@ jobs: - name: Test with pytest run: | - pytest -v -p no:sugar --instafail + pytest -p no:sugar --instafail -vv continue-on-error: false - name: Documentation From 9c7ff1e4918128ff28ba02cb2771b440a392644c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 02:04:52 -0400 Subject: [PATCH 0654/1790] Force pytest color output on CI While pytest-sugar output gets mangled with extra newlines on CI, colorized output seems to work fine and improves readability. --- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 8d1145e76..337c0a809 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -64,4 +64,4 @@ jobs: - name: Test with pytest run: | set +x - /usr/bin/python -m pytest -p no:sugar --instafail -vv + /usr/bin/python -m pytest --color=yes -p no:sugar --instafail -vv diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c0402e4bb..392c894bc 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -78,7 +78,7 @@ jobs: - name: Test with pytest run: | - pytest -p no:sugar --instafail -vv + pytest --color=yes -p no:sugar --instafail -vv continue-on-error: false - name: Documentation From 0eb38bcedf3703c2e3aacae27ea4cbafce33e941 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Sep 2023 06:33:10 -0400 Subject: [PATCH 0655/1790] Fix test_blocking_lock_file for cygwin This permits the longer delay in test_blocking_lock_file--which was already allowed for native Windows--on Cygwin, where it is also needed. That lets the xfail mark for Cygwin be removed. This also updates the comments to avoid implying that the need for the delay is AppVeyor-specific (it seems needed on CI and locally). --- test/test_util.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 42edc57cf..308ba311b 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -12,7 +12,6 @@ from unittest import mock, skipIf from datetime import datetime -import pytest import ddt from git.cmd import dashify @@ -156,11 +155,6 @@ def test_lock_file(self): lock_file._obtain_lock_or_raise() lock_file._release_lock() - @pytest.mark.xfail( - sys.platform == "cygwin", - reason="Cygwin fails here for some reason, always", - raises=AssertionError, - ) def test_blocking_lock_file(self): my_file = tempfile.mktemp() lock_file = BlockingLockFile(my_file) @@ -173,9 +167,8 @@ def test_blocking_lock_file(self): self.assertRaises(IOError, wait_lock._obtain_lock) elapsed = time.time() - start extra_time = 0.02 - if is_win: - # for Appveyor - extra_time *= 6 # NOTE: Indeterministic failures here... + if is_win or sys.platform == "cygwin": + extra_time *= 6 # NOTE: Indeterministic failures without this... self.assertLess(elapsed, wait_time + extra_time) def test_user_id(self): From 715dba473a202ef3631b6c4bd724b8ff4e6c6d0b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Sep 2023 02:51:58 -0400 Subject: [PATCH 0656/1790] Run cygpath tests on Cygwin, not native Windows They were not running on Cygwin, because git.util.is_win is False on Cygwin. They were running on native Windows, with a number of them always failing; these failures had sometimes been obscured by the --maxfail=10 that had formerly been used (from pyproject.toml). Many of them (not all the same ones) fail on Cygwin, and it might be valuable for cygpath to work on other platforms, especially native Windows. But I think it still makes sense to limit the tests to Cygwin at this time, because all the uses of cygpath in the project are in code that only runs after a check that the platform is Cygwin. Part of that check, as it is implemented, explicitly excludes native Windows (is_win must be false). --- test/test_util.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 308ba311b..41d874678 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -9,7 +9,7 @@ import sys import tempfile import time -from unittest import mock, skipIf +from unittest import mock, skipUnless from datetime import datetime import ddt @@ -84,14 +84,14 @@ def setup(self): "array": [42], } - @skipIf(not is_win, "Paths specifically for Windows.") + @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.idata(_norm_cygpath_pairs + _unc_cygpath_pairs) def test_cygpath_ok(self, case): wpath, cpath = case cwpath = cygpath(wpath) self.assertEqual(cwpath, cpath, wpath) - @skipIf(not is_win, "Paths specifically for Windows.") + @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.data( (r"./bar", "bar"), (r".\bar", "bar"), @@ -104,7 +104,7 @@ def test_cygpath_norm_ok(self, case): cwpath = cygpath(wpath) self.assertEqual(cwpath, cpath or wpath, wpath) - @skipIf(not is_win, "Paths specifically for Windows.") + @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.data( r"C:", r"C:Relative", @@ -117,7 +117,7 @@ def test_cygpath_invalids(self, wpath): cwpath = cygpath(wpath) self.assertEqual(cwpath, wpath.replace("\\", "/"), wpath) - @skipIf(not is_win, "Paths specifically for Windows.") + @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.idata(_norm_cygpath_pairs) def test_decygpath(self, case): wpath, cpath = case From d6a2d2807de99715ce85887b8992dbcafcefcee9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 04:08:02 -0400 Subject: [PATCH 0657/1790] Mark some cygpath tests xfail Two of the groups of cygpath tests in test_util.py generate tests that fail on Cygwin. There is no easy way to still run, but xfail, just the specific tests that fail, because the groups of tests are generated with `@ddt` parameterization, but neither the unittest nor pytest xfail mechanisms interact with that. If `@pytest.mark.parametrized` were used, this could be done. But that does not work on methods of test classes that derive from unittest.TestCase, including those in this project that indirectly derive from it by deriving from TestBase. The TestBase base class cannot be removed without overhauling many tests, due to fixtures it provides such as rorepo. So this marks too many tests as xfail, but in doing so allows test runs to pass while still exercising and showing status on all the tests, allowing result changes to be observed easily. --- test/test_util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_util.py b/test/test_util.py index 41d874678..f2135a272 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -13,6 +13,7 @@ from datetime import datetime import ddt +import pytest from git.cmd import dashify from git.compat import is_win @@ -84,6 +85,10 @@ def setup(self): "array": [42], } + @pytest.mark.xfail( + reason="Many return paths prefixed /proc/cygdrive instead.", + raises=AssertionError, + ) @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.idata(_norm_cygpath_pairs + _unc_cygpath_pairs) def test_cygpath_ok(self, case): @@ -91,6 +96,10 @@ def test_cygpath_ok(self, case): cwpath = cygpath(wpath) self.assertEqual(cwpath, cpath, wpath) + @pytest.mark.xfail( + reason=r'2nd example r".\bar" -> "bar" fails, returns "./bar"', + raises=AssertionError, + ) @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.data( (r"./bar", "bar"), From 881456bdceb61d51fa84ea286e6ca0e3587e8dc5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 06:54:56 -0400 Subject: [PATCH 0658/1790] Run test_commit_msg_hook_success on more systems This changes a default Windows skip of test_commit_msg_hook_success to an xfail, and makes it more specific, expecting failure only when either bash.exe is unavailable (definitely expected) or when bash.exe is the WSL bash wrapper in System32, which fails for some reason even though it's not at all clear it ought to. This showcases the failures rather than skipping, and also lets the test pass on Windows systems where bash.exe is something else, including the Git Bash bash.exe that native Windows CI would use. --- test/test_index.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index fba9c78ec..f4858a586 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -7,10 +7,14 @@ from io import BytesIO import os +import os.path as osp +from pathlib import Path from stat import S_ISLNK, ST_MODE +import shutil import tempfile from unittest import skipIf -import shutil + +import pytest from git import ( IndexFile, @@ -23,6 +27,7 @@ GitCommandError, CheckoutError, ) +from git.cmd import Git from git.compat import is_win from git.exc import HookExecutionError, InvalidGitRepositoryError from git.index.fun import hook_path @@ -34,15 +39,22 @@ from git.util import HIDE_WINDOWS_KNOWN_ERRORS, hex_to_bin from gitdb.base import IStream -import os.path as osp -from git.cmd import Git +HOOKS_SHEBANG = "#!/usr/bin/env sh\n" -from pathlib import Path -HOOKS_SHEBANG = "#!/usr/bin/env sh\n" +def _found_in(cmd, directory): + """Check if a command is resolved in a directory (without following symlinks).""" + path = shutil.which(cmd) + return path and Path(path).parent == Path(directory) + is_win_without_bash = is_win and not shutil.which("bash.exe") +is_win_with_wsl_bash = is_win and _found_in( + cmd="bash.exe", + directory=Path(os.getenv("WINDIR")) / "System32", +) + def _make_hook(git_dir, name, content, make_exec=True): """A helper to create a hook""" @@ -910,7 +922,11 @@ def test_pre_commit_hook_fail(self, rw_repo): else: raise AssertionError("Should have caught a HookExecutionError") - @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, "TODO: fix hooks execution on Windows: #703") + @pytest.mark.xfail( + is_win_without_bash or is_win_with_wsl_bash, + reason="Specifically seems to fail on WSL bash (in spite of #1399)", + raises=AssertionError, + ) @with_rw_repo("HEAD", bare=True) def test_commit_msg_hook_success(self, rw_repo): commit_message = "commit default head by Frèderic Çaufl€" From c6a586ab4817a9dcb8c8290a6a3e7071b1834f32 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 07:07:52 -0400 Subject: [PATCH 0659/1790] No longer skip test_index_mutation on Cygwin As it seems to be working now on Cygwin (maybe not native Windows). --- test/test_index.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index f4858a586..06db3aedd 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -12,7 +12,6 @@ from stat import S_ISLNK, ST_MODE import shutil import tempfile -from unittest import skipIf import pytest @@ -27,16 +26,13 @@ GitCommandError, CheckoutError, ) -from git.cmd import Git from git.compat import is_win from git.exc import HookExecutionError, InvalidGitRepositoryError from git.index.fun import hook_path from git.index.typ import BaseIndexEntry, IndexEntry from git.objects import Blob -from test.lib import TestBase, fixture_path, fixture, with_rw_repo -from test.lib import with_rw_directory -from git.util import Actor, rmtree -from git.util import HIDE_WINDOWS_KNOWN_ERRORS, hex_to_bin +from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo +from git.util import Actor, hex_to_bin, rmtree from gitdb.base import IStream HOOKS_SHEBANG = "#!/usr/bin/env sh\n" @@ -434,14 +430,6 @@ def _count_existing(self, repo, files): # END num existing helper - @skipIf( - HIDE_WINDOWS_KNOWN_ERRORS and Git.is_cygwin(), - """FIXME: File "C:\\projects\\gitpython\\git\\test\\test_index.py", line 642, in test_index_mutation - self.assertEqual(fd.read(), link_target) - AssertionError: '!\xff\xfe/\x00e\x00t\x00c\x00/\x00t\x00h\x00a\x00t\x00\x00\x00' - != '/etc/that' - """, - ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): index = rw_repo.index From fc022304d2f4bc8770f75b0c5fc289dccab0ae5b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 24 Sep 2023 02:27:31 -0400 Subject: [PATCH 0660/1790] Report encoding error in test_add_unicode as error This makes the test explicitly error out, rather than skipping, if it appears the environment doesn't support encoding Unicode filenames. Platforms these days should be capable of that, and reporting it as an error lessens the risk of missing a bug in the test code (that method or a fixture) if one is ever introduced. Erroring out will also make it easier to see the details in the chained UnicodeDecodeError exception. This does not affect the behavior of GitPython itself. It only changes how a test reports an unusual condition that keeps the test\ from being usefully run. --- test/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_base.py b/test/test_base.py index b77c8117d..90e701c4b 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -7,7 +7,7 @@ import os import sys import tempfile -from unittest import SkipTest, skipIf +from unittest import skipIf from git import Repo from git.objects import Blob, Tree, Commit, TagObject @@ -126,7 +126,7 @@ def test_add_unicode(self, rw_repo): try: file_path.encode(sys.getfilesystemencoding()) except UnicodeEncodeError as e: - raise SkipTest("Environment doesn't support unicode filenames") from e + raise RuntimeError("Environment doesn't support unicode filenames") from e with open(file_path, "wb") as fp: fp.write(b"something") From 203da23e5fe2ae5a0ae13d0f9b2a276ae584ea7b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 24 Sep 2023 03:19:23 -0400 Subject: [PATCH 0661/1790] Add a few FIXMEs re: better use of xfail --- test/test_config.py | 1 + test/test_util.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_config.py b/test/test_config.py index 481e129c6..f805570d5 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -100,6 +100,7 @@ def test_includes_order(self): # values must be considered as soon as they get them assert r_config.get_value("diff", "tool") == "meld" try: + # FIXME: Split this assertion out somehow and mark it xfail (or fix it). assert r_config.get_value("sec", "var1") == "value1_main" except AssertionError as e: raise SkipTest("Known failure -- included values are not in effect right away") from e diff --git a/test/test_util.py b/test/test_util.py index f2135a272..2b1e518ed 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -85,6 +85,7 @@ def setup(self): "array": [42], } + # FIXME: Mark only the /proc-prefixing cases xfail, somehow (or fix them). @pytest.mark.xfail( reason="Many return paths prefixed /proc/cygdrive instead.", raises=AssertionError, @@ -103,7 +104,7 @@ def test_cygpath_ok(self, case): @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.data( (r"./bar", "bar"), - (r".\bar", "bar"), + (r".\bar", "bar"), # FIXME: Mark only this one xfail, somehow (or fix it). (r"../bar", "../bar"), (r"..\bar", "../bar"), (r"../bar/.\foo/../chu", "../bar/chu"), From cf5f1dca139b809a744badb9f9740dd6d0a70c56 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 03:21:05 -0400 Subject: [PATCH 0662/1790] Report <2.5.1 in test_linked_worktree_traversal as error Although GitPython does not require git >=2.5.1 in general, and this does *not* change that, this makes the unavailability of git 2.5.1 or later an error in test_linked_worktree_traversal, where it is needed to exercise that test, rather than skipping the test. A few systems, such as CentOS 7, may have downstream patched versions of git that remain safe to use yet are numbered <2.5.1 and do not have the necesary feature to run this test. But by now, users of those systems likely anticipate that other software would rely on the presence of features added in git 2.5.1, which was released over 7 years ago. As such, I think it is more useful to give an error for that test, so the test's inability to be run on the system is clear, than to automatically skip the test, which is likely to go unnoticed. --- test/test_fun.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_fun.py b/test/test_fun.py index d76e189ed..f39955aa0 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -2,7 +2,6 @@ from stat import S_IFDIR, S_IFREG, S_IFLNK, S_IXUSR from os import stat import os.path as osp -from unittest import SkipTest from git import Git from git.index import IndexFile @@ -279,7 +278,7 @@ def test_linked_worktree_traversal(self, rw_dir): """Check that we can identify a linked worktree based on a .git file""" git = Git(rw_dir) if git.version_info[:3] < (2, 5, 1): - raise SkipTest("worktree feature unsupported") + raise RuntimeError("worktree feature unsupported (test needs git 2.5.1 or later)") rw_master = self.rorepo.clone(join_path_native(rw_dir, "master_repo")) branch = rw_master.create_head("aaaaaaaa") From 89232365fb0204c32d1f7c23b9eb39fe4401eb7f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 04:13:36 -0400 Subject: [PATCH 0663/1790] Change skipIf(not ...) to skipUnless(...) --- test/test_submodule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 0aa80e5ce..432c02686 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -7,7 +7,7 @@ import tempfile from pathlib import Path import sys -from unittest import mock, skipIf +from unittest import mock, skipIf, skipUnless import pytest @@ -1039,7 +1039,7 @@ def test_branch_renames(self, rw_dir): assert sm_mod.commit() == sm_pfb.commit, "Now head should have been reset" assert sm_mod.head.ref.name == sm_pfb.name - @skipIf(not is_win, "Specifically for Windows.") + @skipUnless(is_win, "Specifically for Windows.") def test_to_relative_path_with_super_at_root_drive(self): class Repo(object): working_tree_dir = "D:\\" From b198bf1e7d9c9d9db675c6c4e94d2f136a0a7923 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 05:43:00 -0400 Subject: [PATCH 0664/1790] Express known test_depth failure with xfail Rather than skipping, so it becomes known if the situation changes. --- test/test_submodule.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 432c02686..54b1e796b 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1050,9 +1050,8 @@ class Repo(object): msg = '_to_relative_path should be "submodule_path" but was "%s"' % relative_path assert relative_path == "submodule_path", msg - @skipIf( - True, - "for some unknown reason the assertion fails, even though it in fact is working in more common setup", + @pytest.mark.xfail( + reason="for some unknown reason the assertion fails, even though it in fact is working in more common setup", ) @with_rw_directory def test_depth(self, rwdir): From cd175a598ed457833bc06adba776e2bbb1d9014b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 07:09:43 -0400 Subject: [PATCH 0665/1790] Remove no-effect `@skipIf` on test_untracked_files It looked like test_untracked_files was sometimes skipped, and specifically that it would be skipped on Cygwin. But the `@skipIf` on it had the condition: HIDE_WINDOWS_KNOWN_ERRORS and Git.is_cygwin() HIDE_WINDOWS_KNOWN_ERRORS can only ever be true if it is set to a truthy value directly (not an intended use as it's a "constant"), or on native Windows systems: no matter how the environment variable related to it is set, it's only checked if is_win, which is set by checking os.name, which is only "nt" on native Windows systems, not Cygwin. So whenever HIDE_WINDOWS_KNOWN_ERRORS is true Git.is_cygwin() will be false. Thus this condition is never true and the test was never being skipped anyway: it was running and passing on Cygwin. --- test/test_repo.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/test_repo.py b/test/test_repo.py index 15899ec50..fe3419e33 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -13,7 +13,7 @@ import pickle import sys import tempfile -from unittest import mock, skipIf, SkipTest, skip +from unittest import mock, SkipTest, skip import pytest @@ -42,7 +42,7 @@ ) from git.repo.fun import touch from test.lib import TestBase, with_rw_repo, fixture -from git.util import HIDE_WINDOWS_KNOWN_ERRORS, cygpath +from git.util import cygpath from test.lib import with_rw_directory from git.util import join_path_native, rmtree, rmfile, bin_to_hex @@ -764,16 +764,6 @@ def test_blame_accepts_rev_opts(self, git): self.rorepo.blame("HEAD", "README.md", rev_opts=["-M", "-C", "-C"]) git.assert_called_once_with(*expected_args, **boilerplate_kwargs) - @skipIf( - HIDE_WINDOWS_KNOWN_ERRORS and Git.is_cygwin(), - """FIXME: File "C:\\projects\\gitpython\\git\\cmd.py", line 671, in execute - raise GitCommandError(command, status, stderr_value, stdout_value) - GitCommandError: Cmd('git') failed due to: exit code(128) - cmdline: git add 1__��ava verb��ten 1_test _myfile 1_test_other_file - 1_��ava-----verb��ten - stderr: 'fatal: pathspec '"1__çava verböten"' did not match any files' - """, - ) @with_rw_repo("HEAD", bare=False) def test_untracked_files(self, rwrepo): for run, repo_add in enumerate((rwrepo.index.add, rwrepo.git.add)): From f38cc000fe4d51f0f9d1bdedec86f6fcdb57f359 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 07:32:46 -0400 Subject: [PATCH 0666/1790] Make 2 more too-low git version skips into errors In the tests only, and not in any way affecting the feature set or requirements of GitPython itself. This is similar to, and with the same reasoning as, cf5f1dc. --- test/test_repo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_repo.py b/test/test_repo.py index fe3419e33..4257d8a47 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -13,7 +13,7 @@ import pickle import sys import tempfile -from unittest import mock, SkipTest, skip +from unittest import mock, skip import pytest @@ -1235,7 +1235,7 @@ def test_merge_base(self): def test_is_ancestor(self): git = self.rorepo.git if git.version_info[:3] < (1, 8, 0): - raise SkipTest("git merge-base --is-ancestor feature unsupported") + raise RuntimeError("git merge-base --is-ancestor feature unsupported (test needs git 1.8.0 or later)") repo = self.rorepo c1 = "f6aa8d1" @@ -1283,7 +1283,7 @@ def test_git_work_tree_dotgit(self, rw_dir): based on it.""" git = Git(rw_dir) if git.version_info[:3] < (2, 5, 1): - raise SkipTest("worktree feature unsupported") + raise RuntimeError("worktree feature unsupported (test needs git 2.5.1 or later)") rw_master = self.rorepo.clone(join_path_native(rw_dir, "master_repo")) branch = rw_master.create_head("aaaaaaaa") From 8fd56e78366470e0b07db48daf623b188e7245ea Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 08:03:58 -0400 Subject: [PATCH 0667/1790] Update test_root_module Windows skip reason The current cause of failure is different from what is documented in the skip reason. --- test/test_submodule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 54b1e796b..4c087caa1 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -477,11 +477,11 @@ def test_base_bare(self, rwrepo): @skipIf( HIDE_WINDOWS_KNOWN_ERRORS, """ - File "C:\\projects\\gitpython\\git\\cmd.py", line 559, in execute - raise GitCommandNotFound(command, err) - git.exc.GitCommandNotFound: Cmd('git') not found due to: OSError('[WinError 6] The handle is invalid') - cmdline: git clone -n --shared -v C:\\projects\\gitpython\\.git Users\\appveyor\\AppData\\Local\\Temp\\1\\tmplyp6kr_rnon_bare_test_root_module - """, # noqa E501 + E PermissionError: + [WinError 32] The process cannot access the file because it is being used by another process: + 'C:\\Users\\ek\\AppData\\Local\\Temp\\non_bare_test_root_modulep0eqt8_r\\git\\ext\\gitdb' + -> 'C:\\Users\\ek\\AppData\\Local\\Temp\\non_bare_test_root_modulep0eqt8_r\\path\\prefix\\git\\ext\\gitdb' + """, ) @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): From c1798f5ead60f74de422732d5229244d00325f24 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 08:06:45 -0400 Subject: [PATCH 0668/1790] Change test_root_module Windows skip to xfail And rewrite the reason to give more useful information. (The new reason also doesn't state the exception type, because that is now specified, and checked by pytest, by being passed as "raises".) --- test/test_submodule.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 4c087caa1..256eb7dbf 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -474,14 +474,13 @@ def test_base_bare(self, rwrepo): reason="Cygwin GitPython can't find submodule SHA", raises=ValueError, ) - @skipIf( + @pytest.mark.xfail( HIDE_WINDOWS_KNOWN_ERRORS, - """ - E PermissionError: - [WinError 32] The process cannot access the file because it is being used by another process: - 'C:\\Users\\ek\\AppData\\Local\\Temp\\non_bare_test_root_modulep0eqt8_r\\git\\ext\\gitdb' - -> 'C:\\Users\\ek\\AppData\\Local\\Temp\\non_bare_test_root_modulep0eqt8_r\\path\\prefix\\git\\ext\\gitdb' - """, + reason=( + '"The process cannot access the file because it is being used by another process"' + + " on first call to rm.update" + ), + raises=PermissionError, ) @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): From ba567521a5a01b93420cab4021d5663af66231ae Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 08:16:05 -0400 Subject: [PATCH 0669/1790] Update test_git_submodules_and_add_sm_with_new_commit skip reason This is working on Cygwin, so that old reason no longer applies. (The test was not being skipped on Cygwin, and was passing.) It is not working on native Windows, due to a PermissionError from attempting to move a file that is still open (which Windows doesn't allow). That may have been the original native Windows skip reason, but the old AppVeyor CI link for it is broken or not public. This makes the reason clear, though maybe I should add more details. --- test/test_submodule.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 256eb7dbf..25b0daa01 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -750,13 +750,12 @@ def test_list_only_valid_submodules(self, rwdir): @skipIf( HIDE_WINDOWS_KNOWN_ERRORS, - """FIXME on cygwin: File "C:\\projects\\gitpython\\git\\cmd.py", line 671, in execute - raise GitCommandError(command, status, stderr_value, stdout_value) - GitCommandError: Cmd('git') failed due to: exit code(128) - cmdline: git add 1__Xava verbXXten 1_test _myfile 1_test_other_file 1_XXava-----verbXXten - stderr: 'fatal: pathspec '"1__çava verböten"' did not match any files' - FIXME on appveyor: see https://ci.appveyor.com/project/Byron/gitpython/build/1.0.185 - """, + """ + E PermissionError: + [WinError 32] The process cannot access the file because it is being used by another process: + 'C:\\Users\\ek\\AppData\\Local\\Temp\\test_git_submodules_and_add_sm_with_new_commitu6d08von\\parent\\module' + -> 'C:\\Users\\ek\\AppData\\Local\\Temp\\test_git_submodules_and_add_sm_with_new_commitu6d08von\\parent\\module_moved' + """ # noqa: E501, ) @with_rw_directory @_patch_git_config("protocol.file.allow", "always") From 8704d1b81ba080cd4baa74968ac4cf84a84e4cbe Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 08:22:32 -0400 Subject: [PATCH 0670/1790] Change test_git_submodules_and_add_sm_with_new_commit Windows skip to xfail And improve details. The xfail is only for native Windows, not Cygwin (same as the old skip was, and still via checking HIDE_WINDOWS_KNOWN_ERRORS). This change is analogous to the change in c1798f5, but for test_git_submodules_and_add_sm_with_new_commit rather than test_root_module. --- test/test_submodule.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 25b0daa01..72d7255b0 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -7,7 +7,7 @@ import tempfile from pathlib import Path import sys -from unittest import mock, skipIf, skipUnless +from unittest import mock, skipUnless import pytest @@ -748,14 +748,13 @@ def test_list_only_valid_submodules(self, rwdir): repo = git.Repo(repo_path) assert len(repo.submodules) == 0 - @skipIf( + @pytest.mark.xfail( HIDE_WINDOWS_KNOWN_ERRORS, - """ - E PermissionError: - [WinError 32] The process cannot access the file because it is being used by another process: - 'C:\\Users\\ek\\AppData\\Local\\Temp\\test_git_submodules_and_add_sm_with_new_commitu6d08von\\parent\\module' - -> 'C:\\Users\\ek\\AppData\\Local\\Temp\\test_git_submodules_and_add_sm_with_new_commitu6d08von\\parent\\module_moved' - """ # noqa: E501, + reason=( + '"The process cannot access the file because it is being used by another process"' + + " on first call to sm.move" + ), + raises=PermissionError, ) @with_rw_directory @_patch_git_config("protocol.file.allow", "always") From 1d6abdca134082bf0bce2e590a52d8c08bf04d9b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 08:34:54 -0400 Subject: [PATCH 0671/1790] Run the tests in test_tree on Windows This stops skipping them, as they are now working. --- test/test_tree.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/test_tree.py b/test/test_tree.py index e59705645..c5ac8d539 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -5,24 +5,14 @@ # the BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO -from unittest import skipIf from git.objects import Tree, Blob from test.lib import TestBase -from git.util import HIDE_WINDOWS_KNOWN_ERRORS import os.path as osp class TestTree(TestBase): - @skipIf( - HIDE_WINDOWS_KNOWN_ERRORS, - """ - File "C:\\projects\\gitpython\\git\\cmd.py", line 559, in execute - raise GitCommandNotFound(command, err) - git.exc.GitCommandNotFound: Cmd('git') not found due to: OSError('[WinError 6] The handle is invalid') - cmdline: git cat-file --batch-check""", - ) def test_serializable(self): # tree at the given commit contains a submodule as well roottree = self.rorepo.tree("6c1faef799095f3990e9970bc2cb10aa0221cf9c") @@ -51,14 +41,6 @@ def test_serializable(self): testtree._deserialize(stream) # END for each item in tree - @skipIf( - HIDE_WINDOWS_KNOWN_ERRORS, - """ - File "C:\\projects\\gitpython\\git\\cmd.py", line 559, in execute - raise GitCommandNotFound(command, err) - git.exc.GitCommandNotFound: Cmd('git') not found due to: OSError('[WinError 6] The handle is invalid') - cmdline: git cat-file --batch-check""", - ) def test_traverse(self): root = self.rorepo.tree("0.1.6") num_recursive = 0 From 5609faa5a1c22654dfc007f7bf229e1b08087aa8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 09:01:31 -0400 Subject: [PATCH 0672/1790] Add missing raises keyword for test_depth xfail I had forgotten to do this earlier when converting from skip to xfail. Besides consistency with the other uses of xfail in the test suite, the benefit of passing "raises" is that pytest checks that the failure gave the expected exception and makes it a non-expected failure if it didn't. --- test/test_submodule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index 72d7255b0..79ff2c5f2 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1049,6 +1049,7 @@ class Repo(object): @pytest.mark.xfail( reason="for some unknown reason the assertion fails, even though it in fact is working in more common setup", + raises=AssertionError, ) @with_rw_directory def test_depth(self, rwdir): From ed95e8e72f6aa697c1ec69595335168e717da013 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 09:07:53 -0400 Subject: [PATCH 0673/1790] Consolidate test_repo module import statements --- test/test_repo.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_repo.py b/test/test_repo.py index 4257d8a47..364b895fb 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -41,10 +41,8 @@ UnsafeProtocolError, ) from git.repo.fun import touch -from test.lib import TestBase, with_rw_repo, fixture -from git.util import cygpath -from test.lib import with_rw_directory -from git.util import join_path_native, rmtree, rmfile, bin_to_hex +from git.util import bin_to_hex, cygpath, join_path_native, rmfile, rmtree +from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo import os.path as osp From ceb4dd3a7e89ac281a6c0d4d815fc13c00cbdf9f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 10:33:23 -0400 Subject: [PATCH 0674/1790] Show more CI system information --- .github/workflows/cygwin-test.yml | 1 + .github/workflows/pythonpackage.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 337c0a809..d0be6bc39 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -55,6 +55,7 @@ jobs: - name: Show version and platform information run: | + /usr/bin/uname -a /usr/bin/git version /usr/bin/python --version /usr/bin/python -c 'import sys; print(sys.platform)' diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 392c894bc..0dc9d217a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -63,6 +63,7 @@ jobs: - name: Show version and platform information run: | + uname -a git version python --version python -c 'import sys; print(sys.platform)' From 3276aac711186a5dd0bd74ba1be8bb6f4ad3d03a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 10:49:50 -0400 Subject: [PATCH 0675/1790] Use Cygwin's bash and git for more CI steps --- .github/workflows/cygwin-test.yml | 34 +++++++++++++++++------------ .github/workflows/pythonpackage.yml | 1 + 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index d0be6bc39..2269df0da 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -14,11 +14,13 @@ jobs: TEMP: "/tmp" defaults: run: - shell: bash.exe --noprofile --norc -exo pipefail -o igncr "{0}" + shell: C:\cygwin\bin\bash.exe --noprofile --norc -exo pipefail -o igncr "{0}" steps: - name: Force LF line endings - run: git config --global core.autocrlf input + run: | + git config --global core.autocrlf input + shell: bash - uses: actions/checkout@v4 with: @@ -29,9 +31,12 @@ jobs: with: packages: python39 python39-pip python39-virtualenv git + - name: Limit $PATH to Cygwin + run: echo 'C:\cygwin\bin' >"$GITHUB_PATH" + - name: Tell git to trust this repo run: | - /usr/bin/git config --global --add safe.directory "$(pwd)" + git config --global --add safe.directory "$(pwd)" - name: Prepare this repo for tests run: | @@ -39,30 +44,31 @@ jobs: - name: Further prepare git configuration for tests run: | - /usr/bin/git config --global user.email "travis@ci.com" - /usr/bin/git config --global user.name "Travis Runner" + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" # If we rewrite the user's config by accident, we will mess it up # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - name: Update PyPA packages run: | - /usr/bin/python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip setuptools wheel - name: Install project and test dependencies run: | - /usr/bin/python -m pip install ".[test]" + python -m pip install ".[test]" - name: Show version and platform information run: | - /usr/bin/uname -a - /usr/bin/git version - /usr/bin/python --version - /usr/bin/python -c 'import sys; print(sys.platform)' - /usr/bin/python -c 'import os; print(os.name)' - /usr/bin/python -c 'import git; print(git.compat.is_win)' + uname -a + command -v git python + git version + python --version + python -c 'import sys; print(sys.platform)' + python -c 'import os; print(os.name)' + python -c 'import git; print(git.compat.is_win)' - name: Test with pytest run: | set +x - /usr/bin/python -m pytest --color=yes -p no:sugar --instafail -vv + python -m pytest --color=yes -p no:sugar --instafail -vv diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0dc9d217a..78d3ddf86 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -64,6 +64,7 @@ jobs: - name: Show version and platform information run: | uname -a + command -v git python git version python --version python -c 'import sys; print(sys.platform)' From 5d4097654c6498d56defcc98dc1611fbfba9b75a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 12:02:35 -0400 Subject: [PATCH 0676/1790] Try to work in all LF on Cygwin CI + Style tweak and comment to clarify the "Limit $PATH" step. --- .github/workflows/cygwin-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 2269df0da..b8f3efbba 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -9,7 +9,6 @@ jobs: fail-fast: false env: CHERE_INVOKING: 1 - SHELLOPTS: igncr TMP: "/tmp" TEMP: "/tmp" defaults: @@ -19,7 +18,7 @@ jobs: steps: - name: Force LF line endings run: | - git config --global core.autocrlf input + git config --global core.autocrlf false # Affects the non-Cygwin git. shell: bash - uses: actions/checkout@v4 @@ -32,11 +31,13 @@ jobs: packages: python39 python39-pip python39-virtualenv git - name: Limit $PATH to Cygwin - run: echo 'C:\cygwin\bin' >"$GITHUB_PATH" + run: | + echo 'C:\cygwin\bin' > "$GITHUB_PATH" # Overwrite it with just this. - - name: Tell git to trust this repo + - name: Special configuration for Cygwin's git run: | git config --global --add safe.directory "$(pwd)" + git config --global core.autocrlf false - name: Prepare this repo for tests run: | @@ -70,5 +71,4 @@ jobs: - name: Test with pytest run: | - set +x python -m pytest --color=yes -p no:sugar --instafail -vv From dda428640113d6a2c30225ea33f1387e784b7289 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Sep 2023 13:42:29 -0400 Subject: [PATCH 0677/1790] Consistent formatting style across all workflows --- .github/workflows/cygwin-test.yml | 3 +++ .github/workflows/lint.yml | 12 +++++++----- .github/workflows/pythonpackage.yml | 3 ++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index b8f3efbba..3cca4dd5f 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -5,12 +5,15 @@ on: [push, pull_request, workflow_dispatch] jobs: build: runs-on: windows-latest + strategy: fail-fast: false + env: CHERE_INVOKING: 1 TMP: "/tmp" TEMP: "/tmp" + defaults: run: shell: C:\cygwin\bin\bash.exe --noprofile --norc -exo pipefail -o igncr "{0}" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5e79664a8..2204bb792 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,8 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.0 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 78d3ddf86..e9ccdd566 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -10,8 +10,8 @@ permissions: jobs: build: - runs-on: ubuntu-latest + strategy: fail-fast: false matrix: @@ -20,6 +20,7 @@ jobs: - experimental: false - python-version: "3.12" experimental: true + defaults: run: shell: /bin/bash --noprofile --norc -exo pipefail {0} From 3007abc6d229bcfe6643963f648597b7e231ab3d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Sep 2023 00:01:32 -0400 Subject: [PATCH 0678/1790] Remove the recently added "Limit $PATH" step I had put that step in the Cygwin workflow for purposes of experimentation, and it seemed to make clearer what is going on, but really it does the opposite: it's deceptive because Cygwin uses other logic to set its PATH. So this step is unnecessary and ineffective at doing what it appears to do. --- .github/workflows/cygwin-test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 3cca4dd5f..35e9fde72 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -33,10 +33,6 @@ jobs: with: packages: python39 python39-pip python39-virtualenv git - - name: Limit $PATH to Cygwin - run: | - echo 'C:\cygwin\bin' > "$GITHUB_PATH" # Overwrite it with just this. - - name: Special configuration for Cygwin's git run: | git config --global --add safe.directory "$(pwd)" From 4860f701b96dc07ac7c489c74c06cae069ae3cd1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Sep 2023 00:14:29 -0400 Subject: [PATCH 0679/1790] Further reduce differences between test workflows This makes the two CI test workflows more similar in a couple of the remaining ways they differ unnecessarily. This could be extended, and otherwise improved upon, in the future. --- .github/workflows/cygwin-test.yml | 5 +++-- .github/workflows/pythonpackage.yml | 10 +++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 35e9fde72..e818803f1 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -42,7 +42,7 @@ jobs: run: | TRAVIS=yes ./init-tests-after-clone.sh - - name: Further prepare git configuration for tests + - name: Set git user identity and command aliases for the tests run: | git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" @@ -52,7 +52,8 @@ jobs: - name: Update PyPA packages run: | - python -m pip install --upgrade pip setuptools wheel + # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + python -m pip install -U pip $(pip freeze --all | grep -oF setuptools) wheel - name: Install project and test dependencies run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e9ccdd566..1b049ba02 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -41,7 +41,7 @@ jobs: run: | TRAVIS=yes ./init-tests-after-clone.sh - - name: Prepare git configuration for tests + - name: Set git user identity and command aliases for the tests run: | git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" @@ -51,12 +51,8 @@ jobs: - name: Update PyPA packages run: | - python -m pip install --upgrade pip wheel - - # Python prior to 3.12 ships setuptools. Upgrade it if present. - if pip freeze --all | grep --quiet '^setuptools=='; then - python -m pip install --upgrade setuptools - fi + # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + python -m pip install -U pip $(pip freeze --all | grep -oF setuptools) wheel - name: Install project and test dependencies run: | From 13b859745de3f5a610e9e6390d55da40c582e663 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Sep 2023 23:41:02 -0400 Subject: [PATCH 0680/1790] Fix new link to license in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dbec36024..c7c2992e3 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ gpg --edit-key 4C08421980C9 ### LICENSE -[New BSD License](https://opensource.org/license/bsd-3-clause/). See the [LICENSE file](https://github.com/gitpython-developers/GitPython/blob/main/license). +[New BSD License](https://opensource.org/license/bsd-3-clause/). See the [LICENSE file][license]. [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md -[license]: https://github.com/gitpython-developers/GitPython/blob/main/license +[license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE From 388c7d1d70c2dabe1fe5e5eb1d453edd8f40a49d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 00:05:08 -0400 Subject: [PATCH 0681/1790] Clearer YAML style for flake8 extra plugin list --- .pre-commit-config.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a34b8af0..67aefb342 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,11 +4,9 @@ repos: hooks: - id: flake8 additional_dependencies: - [ - flake8-bugbear==23.9.16, - flake8-comprehensions==3.14.0, - flake8-typing-imports==1.14.0, - ] + - flake8-bugbear==23.9.16 + - flake8-comprehensions==3.14.0 + - flake8-typing-imports==1.14.0 exclude: ^doc|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks From 1a8f210a42c3f9c46ce3cfa9c25ea39b0a8ca6c6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 02:40:11 -0400 Subject: [PATCH 0682/1790] Drop flake8 suppressions that are no longer needed + Remove the comments that documented those old suppressions + Format the .flake8 file more readably --- .flake8 | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.flake8 b/.flake8 index ed5d036bf..1cc049a69 100644 --- a/.flake8 +++ b/.flake8 @@ -1,38 +1,26 @@ [flake8] + show-source = True -count= True +count = True statistics = True -# E265 = comment blocks like @{ section, which it can't handle + # E266 = too many leading '#' for block comment # E731 = do not assign a lambda expression, use a def -# W293 = Blank line contains whitespace -# W504 = Line break after operator -# E704 = multiple statements in one line - used for @override # TC002 = move third party import to TYPE_CHECKING -# ANN = flake8-annotations # TC, TC2 = flake8-type-checking -# D = flake8-docstrings # select = C,E,F,W ANN, TC, TC2 # to enable code. Disabled if not listed, including builtin codes enable-extensions = TC, TC2 # only needed for extensions not enabled by default -ignore = E265,E266,E731,E704, - W293, W504, - ANN0 ANN1 ANN2, - TC002, - TC0, TC1, TC2 - # B, - A, - D, - RST, RST3 +ignore = E266, E731 -exclude = .tox,.venv,build,dist,doc,git/ext/ +exclude = .tox, .venv, build, dist, doc, git/ext/ rst-roles = # for flake8-RST-docstrings - attr,class,func,meth,mod,obj,ref,term,var # used by sphinx + attr, class, func, meth, mod, obj, ref, term, var # used by sphinx min-python-version = 3.7.0 # for `black` compatibility max-line-length = 120 -extend-ignore = E203,W503 +extend-ignore = E203, W503 From e07d91a6f67c1c4adb897096cb13d6cd1b98cf42 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 1 Oct 2023 15:31:00 -0400 Subject: [PATCH 0683/1790] Drop claim about Cygwin not having git-daemon On a current Cygwin system with git 2.39.0 (the latest version offered by the Cygwin package manager), git-daemon is present, with the Cygwin path /usr/libexec/git-core/git-daemon.exe. In addition, the cygwin-test.yml workflow does not take any special steps to allow git-daemon to work, but all tests pass in it even without skipping or xfailing tests that seem related to git-daemon. The git_daemon_launched function in test/lib/helper.py only invokes git-daemon directly (rather than through "git daemon") when is_win evaluates true, which only happens on native Windows systems, not Cygwin, which is treated the same as (other) Unix-like systems and still works. So maybe Cygwin git-daemon was never a special case. Whether or not it was, the message about git-daemon needing to be findable in a PATH search is also under an is_win check, and thus is never shown on Cygwin. So I've removed the Cygwin part of that message. (Because the path shown is a MinGW-style path, I have kept the wording about that being for MinGW-git, even though it is no longer needed to disambiguate the Cygwin case.) --- README.md | 3 +-- test/lib/helper.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c7c2992e3..65c1e7bae 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,7 @@ executed `git fetch --tags` followed by the `./init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. On _Windows_, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` -exists in `Git\mingw64\libexec\git-core\`; CYGWIN has no daemon, but should get along fine -with MINGW's. +exists in `Git\mingw64\libexec\git-core\`. #### Install test dependencies diff --git a/test/lib/helper.py b/test/lib/helper.py index e8464b7d4..9656345de 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -177,12 +177,10 @@ def git_daemon_launched(base_path, ip, port): gd = None try: if is_win: - ## On MINGW-git, daemon exists in .\Git\mingw64\libexec\git-core\, - # but if invoked as 'git daemon', it detaches from parent `git` cmd, - # and then CANNOT DIE! - # So, invoke it as a single command. - ## Cygwin-git has no daemon. But it can use MINGW's. - # + # On MINGW-git, daemon exists in Git\mingw64\libexec\git-core\, + # but if invoked as 'git daemon', it detaches from parent `git` cmd, + # and then CANNOT DIE! + # So, invoke it as a single command. daemon_cmd = [ "git-daemon", "--enable=receive-pack", @@ -217,12 +215,11 @@ def git_daemon_launched(base_path, ip, port): ) if is_win: msg += textwrap.dedent( - r""" + R""" On Windows, the `git-daemon.exe` must be in PATH. - For MINGW, look into .\Git\mingw64\libexec\git-core\), but problems with paths might appear. - CYGWIN has no daemon, but if one exists, it gets along fine (but has also paths problems).""" + For MINGW, look into \Git\mingw64\libexec\git-core\, but problems with paths might appear.""" ) log.warning(msg, ex, ip, port, base_path, base_path, exc_info=1) From 35e3875cb71913771885646f294cd25cd19c35f3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 1 Oct 2023 17:06:47 -0400 Subject: [PATCH 0684/1790] Allow base_daemon_path to be normalized for Cygwin Since the Cygwin git-daemon can be used. --- test/lib/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 9656345de..1de904610 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -302,7 +302,7 @@ def remote_repo_creator(self): cw.set("url", remote_repo_url) with git_daemon_launched( - Git.polish_url(base_daemon_path, is_cygwin=False), # No daemon in Cygwin. + Git.polish_url(base_daemon_path), "127.0.0.1", GIT_DAEMON_PORT, ): From 5e71c270b2ea0adfc5c0a103fce33ab6acf275b1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 1 Oct 2023 22:43:36 -0400 Subject: [PATCH 0685/1790] Fix the name of the "executes git" test That test is not testing whether or not a shell is used (nor does it need to test that), but just whether the library actually runs git, passes an argument to it, and returns text from its stdout. --- test/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_git.py b/test/test_git.py index 481309538..a21a3c78e 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -73,7 +73,7 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): res = self.git.transform_kwargs(**{"s": True, "t": True}) self.assertEqual({"-s", "-t"}, set(res)) - def test_it_executes_git_to_shell_and_returns_result(self): + def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") def test_it_executes_git_not_from_cwd(self): From 59440607406873a28788ca38526871749c5549f9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 2 Oct 2023 00:17:05 -0400 Subject: [PATCH 0686/1790] Test whether a shell is used In the Popen calls in Git.execute, for all combinations of allowed values for Git.USE_SHELL and the shell= keyword argument. --- test/test_git.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index a21a3c78e..6fd2c8268 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -4,23 +4,24 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ +import contextlib import os +import os.path as osp import shutil import subprocess import sys from tempfile import TemporaryDirectory, TemporaryFile from unittest import mock, skipUnless -from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd -from test.lib import TestBase, fixture_path -from test.lib import with_rw_directory -from git.util import cwd, finalize_process - -import os.path as osp +import ddt +from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd from git.compat import is_win +from git.util import cwd, finalize_process +from test.lib import TestBase, fixture_path, with_rw_directory +@ddt.ddt class TestGit(TestBase): @classmethod def setUpClass(cls): @@ -73,6 +74,28 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): res = self.git.transform_kwargs(**{"s": True, "t": True}) self.assertEqual({"-s", "-t"}, set(res)) + @ddt.data( + (None, False, False), + (None, True, True), + (False, True, False), + (False, False, False), + (True, False, True), + (True, True, True), + ) + @mock.patch.object(cmd, "Popen", wraps=cmd.Popen) # Since it is gotten via a "from" import. + def test_it_uses_shell_or_not_as_specified(self, case, mock_popen): + """A bool passed as ``shell=`` takes precedence over `Git.USE_SHELL`.""" + value_in_call, value_from_class, expected_popen_arg = case + # FIXME: Check what gets logged too! + with mock.patch.object(Git, "USE_SHELL", value_from_class): + with contextlib.suppress(GitCommandError): + self.git.execute( + "git", # No args, so it runs with or without a shell, on all OSes. + shell=value_in_call, + ) + mock_popen.assert_called_once() + self.assertIs(mock_popen.call_args.kwargs["shell"], expected_popen_arg) + def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") From aa5e2f6b24e36d2d38e84e7f2241b104318396c3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 2 Oct 2023 01:45:17 -0400 Subject: [PATCH 0687/1790] Test if whether a shell is used is logged The log message shows "Popen(...)", not "execute(...)". So it should faithfully report what is about to be passed to Popen in cases where a user reading the log would otherwise be misled into thinking this is what has happened. Reporting the actual "shell=" argument passed to Popen is also more useful to log than the argument passed to Git.execute (at least if only one of them is to be logged). --- test/test_git.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 6fd2c8268..389e80552 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -5,8 +5,10 @@ # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib +import logging import os import os.path as osp +import re import shutil import subprocess import sys @@ -74,7 +76,8 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): res = self.git.transform_kwargs(**{"s": True, "t": True}) self.assertEqual({"-s", "-t"}, set(res)) - @ddt.data( + _shell_cases = ( + # value_in_call, value_from_class, expected_popen_arg (None, False, False), (None, True, True), (False, True, False), @@ -82,20 +85,42 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): (True, False, True), (True, True, True), ) + + @ddt.idata(_shell_cases) @mock.patch.object(cmd, "Popen", wraps=cmd.Popen) # Since it is gotten via a "from" import. def test_it_uses_shell_or_not_as_specified(self, case, mock_popen): """A bool passed as ``shell=`` takes precedence over `Git.USE_SHELL`.""" value_in_call, value_from_class, expected_popen_arg = case - # FIXME: Check what gets logged too! + with mock.patch.object(Git, "USE_SHELL", value_from_class): with contextlib.suppress(GitCommandError): self.git.execute( "git", # No args, so it runs with or without a shell, on all OSes. shell=value_in_call, ) + mock_popen.assert_called_once() self.assertIs(mock_popen.call_args.kwargs["shell"], expected_popen_arg) + @ddt.idata(full_case[:2] for full_case in _shell_cases) + @mock.patch.object(cmd, "Popen", wraps=cmd.Popen) # Since it is gotten via a "from" import. + def test_it_logs_if_it_uses_a_shell(self, case, mock_popen): + """``shell=`` in the log message agrees with what is passed to `Popen`.""" + value_in_call, value_from_class = case + + with self.assertLogs(cmd.log, level=logging.DEBUG) as log_watcher: + with mock.patch.object(Git, "USE_SHELL", value_from_class): + with contextlib.suppress(GitCommandError): + self.git.execute( + "git", # No args, so it runs with or without a shell, on all OSes. + shell=value_in_call, + ) + + popen_shell_arg = mock_popen.call_args.kwargs["shell"] + expected_message = re.compile(rf"DEBUG:git.cmd:Popen\(.*\bshell={popen_shell_arg}\b.*\)") + match_attempts = [expected_message.fullmatch(message) for message in log_watcher.output] + self.assertTrue(any(match_attempts), repr(log_watcher.output)) + def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") From 0f19fb0be1bd3ccd3ff8f35dba9e924c9d379e41 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 2 Oct 2023 02:21:10 -0400 Subject: [PATCH 0688/1790] Fix tests so they don't try to run "g" Both new shell-related tests were causing the code under test to split "git" into ["g", "i", "t"] and thus causing the code under test to attempt to execute a "g" command. This passes the command as a one-element list of strings rather than as a string, which avoids this on all operating systems regardless of whether the code under test has a bug being tested for. This would not occur on Windows, which does not iterate commands of type str character-by-character even when the command is run without a shell. But it did happen on other systems. Most of the time, the benefit of using a command that actually runs "git" rather than "g" is the avoidance of confusion in the error message. But this is also important because it is possible for the user who runs the tests to have a "g" command, in which case it could be very inconvenient, or even unsafe, to run "g". This should be avoided even when the code under test has a bug that causes a shell to be used when it shouldn't or not used when it should, so having separate commands (list and str) per test case parameters would not be a sufficient solution: it would only guard against running "g" when a bug in the code under test were absent. --- test/test_git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 389e80552..7e8e7c805 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -95,7 +95,7 @@ def test_it_uses_shell_or_not_as_specified(self, case, mock_popen): with mock.patch.object(Git, "USE_SHELL", value_from_class): with contextlib.suppress(GitCommandError): self.git.execute( - "git", # No args, so it runs with or without a shell, on all OSes. + ["git"], # No args, so it runs with or without a shell, on all OSes. shell=value_in_call, ) @@ -112,7 +112,7 @@ def test_it_logs_if_it_uses_a_shell(self, case, mock_popen): with mock.patch.object(Git, "USE_SHELL", value_from_class): with contextlib.suppress(GitCommandError): self.git.execute( - "git", # No args, so it runs with or without a shell, on all OSes. + ["git"], # No args, so it runs with or without a shell, on all OSes. shell=value_in_call, ) From da3460c6cc3a7c6981dfd1d4675d167a7a5f2b0c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 2 Oct 2023 02:56:37 -0400 Subject: [PATCH 0689/1790] Extract shared test logic to a helper This also helps mock Popen over a smaller scope, which may be beneficial (especially if it is mocked in the subprocess module, rather than the git.cmd module, in the future). --- test/test_git.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 7e8e7c805..343bf7a19 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -86,35 +86,33 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): (True, True, True), ) + def _do_shell_combo(self, value_in_call, value_from_class): + with mock.patch.object(Git, "USE_SHELL", value_from_class): + # git.cmd gets Popen via a "from" import, so patch it there. + with mock.patch.object(cmd, "Popen", wraps=cmd.Popen) as mock_popen: + # Use a command with no arguments (besides the program name), so it runs + # with or without a shell, on all OSes, with the same effect. Since git + # errors out when run with no arguments, we swallow that error. + with contextlib.suppress(GitCommandError): + self.git.execute(["git"], shell=value_in_call) + + return mock_popen + @ddt.idata(_shell_cases) - @mock.patch.object(cmd, "Popen", wraps=cmd.Popen) # Since it is gotten via a "from" import. - def test_it_uses_shell_or_not_as_specified(self, case, mock_popen): + def test_it_uses_shell_or_not_as_specified(self, case): """A bool passed as ``shell=`` takes precedence over `Git.USE_SHELL`.""" value_in_call, value_from_class, expected_popen_arg = case - - with mock.patch.object(Git, "USE_SHELL", value_from_class): - with contextlib.suppress(GitCommandError): - self.git.execute( - ["git"], # No args, so it runs with or without a shell, on all OSes. - shell=value_in_call, - ) - + mock_popen = self._do_shell_combo(value_in_call, value_from_class) mock_popen.assert_called_once() self.assertIs(mock_popen.call_args.kwargs["shell"], expected_popen_arg) @ddt.idata(full_case[:2] for full_case in _shell_cases) - @mock.patch.object(cmd, "Popen", wraps=cmd.Popen) # Since it is gotten via a "from" import. - def test_it_logs_if_it_uses_a_shell(self, case, mock_popen): + def test_it_logs_if_it_uses_a_shell(self, case): """``shell=`` in the log message agrees with what is passed to `Popen`.""" value_in_call, value_from_class = case with self.assertLogs(cmd.log, level=logging.DEBUG) as log_watcher: - with mock.patch.object(Git, "USE_SHELL", value_from_class): - with contextlib.suppress(GitCommandError): - self.git.execute( - ["git"], # No args, so it runs with or without a shell, on all OSes. - shell=value_in_call, - ) + mock_popen = self._do_shell_combo(value_in_call, value_from_class) popen_shell_arg = mock_popen.call_args.kwargs["shell"] expected_message = re.compile(rf"DEBUG:git.cmd:Popen\(.*\bshell={popen_shell_arg}\b.*\)") From 41294d578471f7f63c02bf59e8abc3459f9d6390 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 2 Oct 2023 03:26:02 -0400 Subject: [PATCH 0690/1790] Use the mock backport on Python 3.7 Because mock.call.kwargs, i.e. the ability to examine m.call_args.kwargs where m is a Mock or MagicMock, was introduced in Python 3.8. Currently it is only in test/test_git.py that any use of mocks requires this, so I've put the conditional import logic to import mock (the top-level package) rather than unittest.mock only there. The mock library is added as a development (testing) dependency only when the Python version is lower than 3.8, so it is not installed when not needed. This fixes a problem in the new tests of whether a shell is used, and reported as used, in the Popen call in Git.execute. Those just-introduced tests need this, to be able to use mock_popen.call_args.kwargs on Python 3.7. --- test-requirements.txt | 3 ++- test/test_git.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1c08c736f..9414da09c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,7 @@ black coverage[toml] -ddt>=1.1.1, !=1.4.3 +ddt >= 1.1.1, != 1.4.3 +mock ; python_version < "3.8" mypy pre-commit pytest diff --git a/test/test_git.py b/test/test_git.py index 343bf7a19..583c74fa3 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -13,7 +13,12 @@ import subprocess import sys from tempfile import TemporaryDirectory, TemporaryFile -from unittest import mock, skipUnless +from unittest import skipUnless + +if sys.version_info >= (3, 8): + from unittest import mock +else: + import mock # To be able to examine call_args.kwargs on a mock. import ddt From baf3457ec8f92c64d2481b812107e6acc4059ddd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 2 Oct 2023 04:11:18 -0400 Subject: [PATCH 0691/1790] Fix Git.execute shell use and reporting bugs This updates the shell variable itself, only when it is None, from self.USE_SHELL. (That attribute is usually Git.USE_SHELL rather than an instance attribute. But although people probably shouldn't set it on instances, people will expect that this is permitted.) Now: - USE_SHELL is used as a fallback only. When shell=False is passed, USE_SHELL is no longer consuled. Thus shell=False always keeps a shell from being used, even in the non-default case where the USE_SHELL attribue has been set to True. - The debug message printed to the log shows the actual value that is being passed to Popen, because the updated shell variable is used both to produce that message and in the Popen call. --- git/cmd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 9921dd6c9..53b8b9050 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -974,6 +974,8 @@ def execute( istream_ok = "None" if istream: istream_ok = "" + if shell is None: + shell = self.USE_SHELL log.debug( "Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", redacted_command, @@ -992,7 +994,7 @@ def execute( stdin=istream or DEVNULL, stderr=PIPE, stdout=stdout_sink, - shell=shell is not None and shell or self.USE_SHELL, + shell=shell, close_fds=is_posix, # unsupported on windows universal_newlines=universal_newlines, creationflags=PROC_CREATIONFLAGS, From b79198a880982e6768fec4d0ef244338420efbdc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 04:58:13 -0400 Subject: [PATCH 0692/1790] Document Git.execute parameters in definition order - Reorder the items in the git.cmd.Git.execute docstring that document its parameters, to be in the same order the parameters are actually defined in. - Use consistent spacing, by having a blank line between successive items that document parameters. Before, most of them were separated this way, but some of them were not. - Reorder the elements of execute_kwargs (which list all those parameters except the first parameter, command) to be listed in that order as well. These were mostly in order, but a couple were out of order. This is just about the order they appear in the definition, since sets in Python (unlike dicts) have no key order guarantees. --- git/cmd.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 53b8b9050..4d772e909 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -66,10 +66,10 @@ "with_extended_output", "with_exceptions", "as_process", - "stdout_as_string", "output_stream", - "with_stdout", + "stdout_as_string", "kill_after_timeout", + "with_stdout", "universal_newlines", "shell", "env", @@ -883,6 +883,27 @@ def execute( decoded into a string using the default encoding (usually utf-8). The latter can fail, if the output contains binary data. + :param kill_after_timeout: + To specify a timeout in seconds for the git command, after which the process + should be killed. This will have no effect if as_process is set to True. It is + set to None by default and will let the process run until the timeout is + explicitly specified. This feature is not supported on Windows. It's also worth + noting that kill_after_timeout uses SIGKILL, which can have negative side + effects on a repository. For example, stale locks in case of git gc could + render the repository incapable of accepting changes until the lock is manually + removed. + + :param with_stdout: + If True, default True, we open stdout on the created process + + :param universal_newlines: + if True, pipes will be opened as text, and lines are split at + all known line endings. + + :param shell: + Whether to invoke commands through a shell (see `Popen(..., shell=True)`). + It overrides :attr:`USE_SHELL` if it is not `None`. + :param env: A dictionary of environment variables to be passed to `subprocess.Popen`. @@ -891,29 +912,14 @@ def execute( one invocation of write() method. If the given number is not positive then the default value is used. + :param strip_newline_in_stdout: + Whether to strip the trailing ``\\n`` of the command stdout. + :param subprocess_kwargs: Keyword arguments to be passed to subprocess.Popen. Please note that some of the valid kwargs are already set by this method, the ones you specify may not be the same ones. - :param with_stdout: If True, default True, we open stdout on the created process - :param universal_newlines: - if True, pipes will be opened as text, and lines are split at - all known line endings. - :param shell: - Whether to invoke commands through a shell (see `Popen(..., shell=True)`). - It overrides :attr:`USE_SHELL` if it is not `None`. - :param kill_after_timeout: - To specify a timeout in seconds for the git command, after which the process - should be killed. This will have no effect if as_process is set to True. It is - set to None by default and will let the process run until the timeout is - explicitly specified. This feature is not supported on Windows. It's also worth - noting that kill_after_timeout uses SIGKILL, which can have negative side - effects on a repository. For example, stale locks in case of git gc could - render the repository incapable of accepting changes until the lock is manually - removed. - :param strip_newline_in_stdout: - Whether to strip the trailing ``\\n`` of the command stdout. :return: * str(output) if extended_output = False (Default) * tuple(int(status), str(stdout), str(stderr)) if extended_output = True From 13e1593fc6a3c218451e29dd6b4a58b3a44afee3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 05:30:22 -0400 Subject: [PATCH 0693/1790] Don't say Git.execute uses a shell, in its summary The top line of the Git.execute docstring said that it used a shell, which is not necessarily the case (and is not usually the case, since the default is not to use one). This removes that claim while keeping the top-line wording otherwise the same. It also rephrases the description of the command parameter, in a way that does not change its meaning but reflects the more common practice of passing a sequence of arguments (since portable calls that do not use a shell must do that). --- git/cmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 4d772e909..e324db7e2 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -842,12 +842,12 @@ def execute( strip_newline_in_stdout: bool = True, **subprocess_kwargs: Any, ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], AutoInterrupt]: - """Handles executing the command on the shell and consumes and returns - the returned information (stdout) + """Handles executing the command and consumes and returns the returned + information (stdout) :param command: The command argument list to execute. - It should be a string, or a sequence of program arguments. The + It should be a sequence of program arguments, or a string. The program to execute is the first item in the args sequence or string. :param istream: From 74b68e9b621a729a3407b2020b0a48d7921fb1e9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 05:55:54 -0400 Subject: [PATCH 0694/1790] Copyedit Git.execute docstring These are some small clarity and consistency revisions to the docstring of git.cmd.Git.execute that didn't specifically fit in the narrow topics of either of the preceding two commits. --- git/cmd.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index e324db7e2..f20cd77af 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -851,7 +851,7 @@ def execute( program to execute is the first item in the args sequence or string. :param istream: - Standard input filehandle passed to subprocess.Popen. + Standard input filehandle passed to `subprocess.Popen`. :param with_extended_output: Whether to return a (status, stdout, stderr) tuple. @@ -862,8 +862,7 @@ def execute( :param as_process: Whether to return the created process instance directly from which streams can be read on demand. This will render with_extended_output and - with_exceptions ineffective - the caller will have - to deal with the details himself. + with_exceptions ineffective - the caller will have to deal with the details. It is important to note that the process will be placed into an AutoInterrupt wrapper that will interrupt the process once it goes out of scope. If you use the command in iterators, you should pass the whole process instance @@ -876,25 +875,25 @@ def execute( always be created with a pipe due to issues with subprocess. This merely is a workaround as data will be copied from the output pipe to the given output stream directly. - Judging from the implementation, you shouldn't use this flag ! + Judging from the implementation, you shouldn't use this parameter! :param stdout_as_string: - if False, the commands standard output will be bytes. Otherwise, it will be - decoded into a string using the default encoding (usually utf-8). + If False, the command's standard output will be bytes. Otherwise, it will be + decoded into a string using the default encoding (usually UTF-8). The latter can fail, if the output contains binary data. :param kill_after_timeout: - To specify a timeout in seconds for the git command, after which the process + Specifies a timeout in seconds for the git command, after which the process should be killed. This will have no effect if as_process is set to True. It is set to None by default and will let the process run until the timeout is explicitly specified. This feature is not supported on Windows. It's also worth noting that kill_after_timeout uses SIGKILL, which can have negative side - effects on a repository. For example, stale locks in case of git gc could + effects on a repository. For example, stale locks in case of ``git gc`` could render the repository incapable of accepting changes until the lock is manually removed. :param with_stdout: - If True, default True, we open stdout on the created process + If True, default True, we open stdout on the created process. :param universal_newlines: if True, pipes will be opened as text, and lines are split at @@ -916,19 +915,19 @@ def execute( Whether to strip the trailing ``\\n`` of the command stdout. :param subprocess_kwargs: - Keyword arguments to be passed to subprocess.Popen. Please note that - some of the valid kwargs are already set by this method, the ones you + Keyword arguments to be passed to `subprocess.Popen`. Please note that + some of the valid kwargs are already set by this method; the ones you specify may not be the same ones. :return: * str(output) if extended_output = False (Default) * tuple(int(status), str(stdout), str(stderr)) if extended_output = True - if output_stream is True, the stdout value will be your output stream: + If output_stream is True, the stdout value will be your output stream: * output_stream if extended_output = False * tuple(int(status), output_stream, str(stderr)) if extended_output = True - Note git is executed with LC_MESSAGES="C" to ensure consistent + Note that git is executed with ``LC_MESSAGES="C"`` to ensure consistent output regardless of system language. :raise GitCommandError: From 133271bb3871baff3ed772fbdea8bc359f115fd6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 06:34:05 -0400 Subject: [PATCH 0695/1790] Other copyediting in the git.cmd module (Not specific to git.cmd.Git.execute.) --- git/cmd.py | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index f20cd77af..a6d287986 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -105,7 +105,7 @@ def handle_process_output( ) -> None: """Registers for notifications to learn that process output is ready to read, and dispatches lines to the respective line handlers. - This function returns once the finalizer returns + This function returns once the finalizer returns. :return: result of finalizer :param process: subprocess.Popen instance @@ -294,9 +294,7 @@ def __setstate__(self, d: Dict[str, Any]) -> None: @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: - """This gets called by the refresh function (see the top level - __init__). - """ + """This gets called by the refresh function (see the top level __init__).""" # discern which path to refresh with if path is not None: new_git = os.path.expanduser(path) @@ -446,9 +444,9 @@ def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: if is_cygwin: url = cygpath(url) else: - """Remove any backslahes from urls to be written in config files. + """Remove any backslashes from urls to be written in config files. - Windows might create config-files containing paths with backslashed, + Windows might create config files containing paths with backslashes, but git stops liking them as it will escape the backslashes. Hence we undo the escaping just to be sure. """ @@ -464,8 +462,8 @@ def check_unsafe_protocols(cls, url: str) -> None: Check for unsafe protocols. Apart from the usual protocols (http, git, ssh), - Git allows "remote helpers" that have the form `::

`, - one of these helpers (`ext::`) can be used to invoke any arbitrary command. + Git allows "remote helpers" that have the form ``::
``, + one of these helpers (``ext::``) can be used to invoke any arbitrary command. See: @@ -517,7 +515,7 @@ def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: self.status: Union[int, None] = None def _terminate(self) -> None: - """Terminate the underlying process""" + """Terminate the underlying process.""" if self.proc is None: return @@ -572,7 +570,7 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: """Wait for the process and return its status code. :param stderr: Previously read value of stderr, in case stderr is already closed. - :warn: may deadlock if output or error pipes are used and not handled separately. + :warn: May deadlock if output or error pipes are used and not handled separately. :raise GitCommandError: if the return status is not 0""" if stderr is None: stderr_b = b"" @@ -605,13 +603,12 @@ def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> byte # END auto interrupt class CatFileContentStream(object): - """Object representing a sized read-only stream returning the contents of an object. It behaves like a stream, but counts the data read and simulates an empty stream once our sized content region is empty. - If not all data is read to the end of the objects's lifetime, we read the - rest to assure the underlying stream continues to work""" + If not all data is read to the end of the object's lifetime, we read the + rest to assure the underlying stream continues to work.""" __slots__: Tuple[str, ...] = ("_stream", "_nbr", "_size") @@ -740,11 +737,11 @@ def __getattr__(self, name: str) -> Any: def set_persistent_git_options(self, **kwargs: Any) -> None: """Specify command line options to the git executable - for subsequent subcommand calls + for subsequent subcommand calls. :param kwargs: is a dict of keyword arguments. - these arguments are passed as in _call_process + These arguments are passed as in _call_process but will be passed to the git command rather than the subcommand. """ @@ -775,7 +772,7 @@ def version_info(self) -> Tuple[int, int, int, int]: """ :return: tuple(int, int, int, int) tuple with integers representing the major, minor and additional version numbers as parsed from git version. - This value is generated on demand and is cached""" + This value is generated on demand and is cached.""" return self._version_info @overload @@ -843,7 +840,7 @@ def execute( **subprocess_kwargs: Any, ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], AutoInterrupt]: """Handles executing the command and consumes and returns the returned - information (stdout) + information (stdout). :param command: The command argument list to execute. @@ -1213,7 +1210,7 @@ def _unpack_args(cls, arg_list: Sequence[str]) -> List[str]: def __call__(self, **kwargs: Any) -> "Git": """Specify command line options to the git executable - for a subcommand call + for a subcommand call. :param kwargs: is a dict of keyword arguments. @@ -1251,7 +1248,7 @@ def _call_process( self, method: str, *args: Any, **kwargs: Any ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: """Run the given git command with the specified arguments and return - the result as a String + the result as a string. :param method: is the command. Contained "_" characters will be converted to dashes, @@ -1260,7 +1257,7 @@ def _call_process( :param args: is the list of arguments. If None is included, it will be pruned. This allows your commands to call git more conveniently as None - is realized as non-existent + is realized as non-existent. :param kwargs: It contains key-values for the following: @@ -1390,7 +1387,7 @@ def get_object_header(self, ref: str) -> Tuple[str, str, int]: return self.__get_object_header(cmd, ref) def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: - """As get_object_header, but returns object data as well + """As get_object_header, but returns object data as well. :return: (hexsha, type_string, size_as_int, data_string) :note: not threadsafe""" @@ -1400,10 +1397,10 @@ def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: return (hexsha, typename, size, data) def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileContentStream"]: - """As get_object_header, but returns the data as a stream + """As get_object_header, but returns the data as a stream. :return: (hexsha, type_string, size_as_int, stream) - :note: This method is not threadsafe, you need one independent Command instance per thread to be safe !""" + :note: This method is not threadsafe, you need one independent Command instance per thread to be safe!""" cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) cmd_stdout = cmd.stdout if cmd.stdout is not None else io.BytesIO() From fc755dae6866b9c9e0aa347297b693fec2c5b415 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 06:38:54 -0400 Subject: [PATCH 0696/1790] Avoid having a local function seem to be a method The kill_process local function defined in the Git.execute method is a local function and not a method, but it was easy to misread as a method for two reasons: - Its docstring described it as a method. - It was named with a leading underscore, as though it were a nonpublic method. But this name is a local variable, and local variables are always nonpublic (except when they are function parameters, in which case they are in a sense public). A leading underscore in a local variable name usually means the variable is unused in the function. This fixes the docstring and drops the leading underscore from the name. If this is ever extracted from the Git.execute method and placed at class or module scope, then the name can be changed back. --- git/cmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index a6d287986..1d8c70f32 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1012,8 +1012,8 @@ def execute( if as_process: return self.AutoInterrupt(proc, command) - def _kill_process(pid: int) -> None: - """Callback method to kill a process.""" + def kill_process(pid: int) -> None: + """Callback to kill a process.""" p = Popen( ["ps", "--ppid", str(pid)], stdout=PIPE, @@ -1046,7 +1046,7 @@ def _kill_process(pid: int) -> None: if kill_after_timeout is not None: kill_check = threading.Event() - watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid,)) + watchdog = threading.Timer(kill_after_timeout, kill_process, args=(proc.pid,)) # Wait for the process to return status = 0 From 2d1efdca84e266a422f4298ee94ee9b8dae6c32e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 07:55:35 -0400 Subject: [PATCH 0697/1790] Test that git.cmd.execute_kwargs is correct --- test/test_git.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_git.py b/test/test_git.py index 583c74fa3..723bf624b 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -5,6 +5,7 @@ # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib +import inspect import logging import os import os.path as osp @@ -364,3 +365,11 @@ def counter_stderr(line): self.assertEqual(count[1], line_count) self.assertEqual(count[2], line_count) + + def test_execute_kwargs_set_agrees_with_method(self): + parameter_names = inspect.signature(cmd.Git.execute).parameters.keys() + self_param, command_param, *most_params, extra_kwargs_param = parameter_names + self.assertEqual(self_param, "self") + self.assertEqual(command_param, "command") + self.assertEqual(set(most_params), cmd.execute_kwargs) # Most important. + self.assertEqual(extra_kwargs_param, "subprocess_kwargs") From a8a43fe6f8d6f0a7f9067149859634d624406bb1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 08:50:17 -0400 Subject: [PATCH 0698/1790] Simplify shell test helper with with_exceptions=False Instead of swallowing GitCommandError exceptions in the helper used by test_it_uses_shell_or_not_as_specified and test_it_logs_if_it_uses_a_shell, this modifies the helper so it prevents Git.execute from raising the exception in the first place. --- test/test_git.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 723bf624b..8d269f38a 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -4,7 +4,6 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ -import contextlib import inspect import logging import os @@ -97,10 +96,8 @@ def _do_shell_combo(self, value_in_call, value_from_class): # git.cmd gets Popen via a "from" import, so patch it there. with mock.patch.object(cmd, "Popen", wraps=cmd.Popen) as mock_popen: # Use a command with no arguments (besides the program name), so it runs - # with or without a shell, on all OSes, with the same effect. Since git - # errors out when run with no arguments, we swallow that error. - with contextlib.suppress(GitCommandError): - self.git.execute(["git"], shell=value_in_call) + # with or without a shell, on all OSes, with the same effect. + self.git.execute(["git"], with_exceptions=False, shell=value_in_call) return mock_popen From 9fa1ceef2c47c9f58e10d8925cc166fdfd6b5183 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 09:45:45 -0400 Subject: [PATCH 0699/1790] Extract a _assert_logged_for_popen method This extracts the logic of searching log messages, and asserting that (at least) one matches a pattern for the report of a Popen call with a given argument, from test_it_logs_if_it_uses_a_shell into a new nonpublic test helper method _assert_logged_for_popen. The extracted version is modified to make it slightly more general, and slightly more robust. This is still not extremely robust: the notation used to log Popen calls is informal, so it wouldn't make sense to really parse it as code. But this no longer assumes that the representation of a value ends at a word boundary, nor that the value is free of regular expression metacharacters. --- test/test_git.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 8d269f38a..10b346e4d 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -40,6 +40,13 @@ def tearDown(self): gc.collect() + def _assert_logged_for_popen(self, log_watcher, name, value): + re_name = re.escape(name) + re_value = re.escape(str(value)) + re_line = re.compile(fr"DEBUG:git.cmd:Popen\(.*\b{re_name}={re_value}[,)]") + match_attempts = [re_line.match(message) for message in log_watcher.output] + self.assertTrue(any(match_attempts), repr(log_watcher.output)) + @mock.patch.object(Git, "execute") def test_call_process_calls_execute(self, git): git.return_value = "" @@ -113,14 +120,9 @@ def test_it_uses_shell_or_not_as_specified(self, case): def test_it_logs_if_it_uses_a_shell(self, case): """``shell=`` in the log message agrees with what is passed to `Popen`.""" value_in_call, value_from_class = case - with self.assertLogs(cmd.log, level=logging.DEBUG) as log_watcher: mock_popen = self._do_shell_combo(value_in_call, value_from_class) - - popen_shell_arg = mock_popen.call_args.kwargs["shell"] - expected_message = re.compile(rf"DEBUG:git.cmd:Popen\(.*\bshell={popen_shell_arg}\b.*\)") - match_attempts = [expected_message.fullmatch(message) for message in log_watcher.output] - self.assertTrue(any(match_attempts), repr(log_watcher.output)) + self._assert_logged_for_popen(log_watcher, "shell", mock_popen.call_args.kwargs["shell"]) def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") From 790a790f49a2548c620532ee2b9705b09fb33855 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 09:59:28 -0400 Subject: [PATCH 0700/1790] Log stdin arg as such, and test that this is done This changes how the Popen call debug logging line shows the informal summary of what kind of thing is being passed as the stdin argument to Popen, showing it with stdin= rather than istream=. The new test, with "istream" in place of "stdin", passed before the code change in the git.cmd module, failed once "istream" was changed to "stdin" in the test, and then, as expected, passed again once "istream=" was changed to "stdin=" in the log.debug call in git.cmd.Git.execute. --- git/cmd.py | 2 +- test/test_git.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 1d8c70f32..e545ba160 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -979,7 +979,7 @@ def execute( if shell is None: shell = self.USE_SHELL log.debug( - "Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", + "Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, stdin=%s)", redacted_command, cwd, universal_newlines, diff --git a/test/test_git.py b/test/test_git.py index 10b346e4d..1ee7b3642 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -124,6 +124,16 @@ def test_it_logs_if_it_uses_a_shell(self, case): mock_popen = self._do_shell_combo(value_in_call, value_from_class) self._assert_logged_for_popen(log_watcher, "shell", mock_popen.call_args.kwargs["shell"]) + @ddt.data( + ("None", None), + ("", subprocess.PIPE), + ) + def test_it_logs_istream_summary_for_stdin(self, case): + expected_summary, istream_argument = case + with self.assertLogs(cmd.log, level=logging.DEBUG) as log_watcher: + self.git.execute(["git", "version"], istream=istream_argument) + self._assert_logged_for_popen(log_watcher, "stdin", expected_summary) + def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") From c3fde7fb8dcd48d17ee24b78db7b0dd25d2348ab Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 10:16:10 -0400 Subject: [PATCH 0701/1790] Log args in the order they are passed to Popen This is still not including all or even most of the arguments, nor are all the logged arguments literal (nor should either of those things likely be changed). It is just to facilitate easier comparison of what is logged to the Popen call in the code. --- git/cmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index e545ba160..c35e56703 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -979,12 +979,12 @@ def execute( if shell is None: shell = self.USE_SHELL log.debug( - "Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, stdin=%s)", + "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, cwd, - universal_newlines, - shell, istream_ok, + shell, + universal_newlines, ) try: with maybe_patch_caller_env: From ab958865d89b3146bb953f826f30da11dc59139a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 10:30:58 -0400 Subject: [PATCH 0702/1790] Eliminate istream_ok variable In Git.execute, the stdin argument to Popen is the only one where a compound expression (rather than a single term) is currently passed. So having that be the same in the log message makes it easier to understand what is going on, as well as to see how the information shown in the log corresponds to what Popen receives. --- git/cmd.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index c35e56703..7c448e3f2 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -973,16 +973,13 @@ def execute( # end handle stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") - istream_ok = "None" - if istream: - istream_ok = "" if shell is None: shell = self.USE_SHELL log.debug( "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, cwd, - istream_ok, + "" if istream else "None", shell, universal_newlines, ) From c3e926fbfda3bf5a1723258e990dfcfa45f2ef86 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 12:16:07 -0400 Subject: [PATCH 0703/1790] Fix a small YAML formatting style inconsistency --- .github/workflows/pythonpackage.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 1b049ba02..5564a526b 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,9 +17,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - - experimental: false - - python-version: "3.12" - experimental: true + - experimental: false + - python-version: "3.12" + experimental: true defaults: run: From b54a28a50b3e81e04b5b9ef61297fd12c62d09b3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 12:17:58 -0400 Subject: [PATCH 0704/1790] No longer allow CI to select a prerelease for 3.12 Since 3.12.0 stable is out, and available via setup-python, per: https://github.com/actions/python-versions/blob/main/versions-manifest.json --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 5564a526b..e43317807 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,8 +18,6 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - - python-version: "3.12" - experimental: true defaults: run: From cd8da415e119451e195903576a8ff22cf60ae47b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 12:42:23 -0400 Subject: [PATCH 0705/1790] Run CI on all branches This makes it easier to test changes to CI without/before a PR. + Fix a small YAML indentation style inconsistency. --- .github/workflows/pythonpackage.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6849763e1..6d819ff42 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -3,11 +3,7 @@ name: Python package -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [push, pull_request, workflow_dispatch] jobs: build: @@ -17,9 +13,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - - experimental: false - - python-version: "3.12" - experimental: true + - experimental: false + - python-version: "3.12" + experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 00bbc44eff7be0b43a882eadf54c2bbb2f9000a0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 12:44:38 -0400 Subject: [PATCH 0706/1790] No longer treat 3.12 as experimental on CI Since Python 3.12.0 stable has been released, as well as now being available via setup-python, per: https://github.com/actions/python-versions/blob/main/versions-manifest.json The main practical effect of this is that continue-on-error is no longer set to true for 3.12, so 3.12 is no longer special-cased to refrain from cancelling other test jobs when its test job fails. Another effect is that 3.12 can longer be selected as a prerelease. --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6d819ff42..73b3902c3 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,8 +14,6 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - - python-version: "3.12" - experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 69cd90ff968bd466b26d52859e5a850b6ce300b1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 13:09:45 -0400 Subject: [PATCH 0707/1790] Run CI on all branches + Make YAML indentation more consistent. --- .github/workflows/pythonpackage.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7e21bb9a7..28054ea52 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -3,11 +3,7 @@ name: Python package -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [push, pull_request, workflow_dispatch] jobs: build: @@ -17,9 +13,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - - experimental: false - - python-version: "3.12" - experimental: true + - experimental: false + - python-version: "3.12" + experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 44ac30a7481a619e32b13a4efc1de0479b1b4379 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 13:11:57 -0400 Subject: [PATCH 0708/1790] No longer treat 3.12 as experimental on CI Since Python 3.12.0 stable has been released and is available via setup-python, per: https://github.com/actions/python-versions/blob/main/versions-manifest.json This makes it so continue-on-error and allow-prereleases are no longer special-cased to true for 3.12; instead, 3.12 is treaed the same as other Python releases (where those are false). --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 28054ea52..e816355f3 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,8 +14,6 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - - python-version: "3.12" - experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 8c4df3cfdca1eebd2f07c7be24ab5e2805ec2708 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 03:12:48 -0400 Subject: [PATCH 0709/1790] Add pre-commit hook to run shellcheck This also reorders the hooks from pre-commit/pre-commit-hooks so that the overall order of all hooks from all repositories is: lint Python, lint non-Python, non-lint. --- .pre-commit-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67aefb342..c5973c9ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,9 +9,14 @@ repos: - flake8-typing-imports==1.14.0 exclude: ^doc|^git/ext/ + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.5 + hooks: + - id: shellcheck + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - - id: check-merge-conflict - id: check-toml - id: check-yaml + - id: check-merge-conflict From f3be76f474636f9805756bc9f05b22fb4aa8809d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 04:49:28 -0400 Subject: [PATCH 0710/1790] Force color when running shellcheck in pre-commit Its output is colorized normally, but on CI it is not colorized without this (pre-commit's own output is, but not shellcheck's). --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5973c9ea..bacc90913 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,7 @@ repos: rev: v0.9.0.5 hooks: - id: shellcheck + args: [--color] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 From 7dd8added2b1695b1740f0d1d7d7b2858a49a88c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 04:17:04 -0400 Subject: [PATCH 0711/1790] Suppress SC2086 where word splitting is intended This suppresses ShellCheck SC2016, "Double quote to prevent globbing and word splitting," on the command in the version check script that expands $config_opts to build the "-c ..." arguments. It also moves the code repsonsible for getting the latest tag, which this is part of, into a function for that purpose, so it's clear that building config_opts is specifically for that, and so that the code is not made harder to read by adding the ShellCheck suppression comment. (The suppression applies only to the immediate next command.) --- check-version.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/check-version.sh b/check-version.sh index c50bf498b..46aa56173 100755 --- a/check-version.sh +++ b/check-version.sh @@ -10,6 +10,13 @@ trap 'echo "$0: Check failed. Stopping." >&2' ERR readonly version_path='VERSION' readonly changes_path='doc/source/changes.rst' +function get_latest_tag() { + local config_opts + config_opts="$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" + # shellcheck disable=SC2086 # Deliberately word-splitting the arguments. + git $config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1 +} + echo 'Checking current directory.' test "$(cd -- "$(dirname -- "$0")" && pwd)" = "$(pwd)" # Ugly, but portable. @@ -26,8 +33,7 @@ test -z "$(git status -s --ignore-submodules)" version_version="$(cat "$version_path")" changes_version="$(awk '/^[0-9]/ {print $0; exit}' "$changes_path")" -config_opts="$(printf ' -c versionsort.suffix=-%s' alpha beta pre rc RC)" -latest_tag="$(git $config_opts tag -l '[0-9]*' --sort=-v:refname | head -n1)" +latest_tag="$(get_latest_tag)" head_sha="$(git rev-parse HEAD)" latest_tag_sha="$(git rev-parse "${latest_tag}^{commit}")" From 21875b5a84c899a5b38c627e895a1bb58344b2a1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 04:31:18 -0400 Subject: [PATCH 0712/1790] Don't split and glob the interpreter name In the release building script, this changes $1 to "$1", because $1 without quotes means to perform word splitting and globbing (pathname expansion) on the parameter (unless otherwise disabled by the value of $IFS or "set -f", respectively) and use the result of doing so, which isn't the intent of the code. This function is only used from within the script, where it is not given values that would be changed by these additional expansions. So this is mainly about communicating intent. (If in the future it is intended that multiple arguments be usable, then they should be passed as separate arguments to release_with, which should be modified by replacing "$1" with "$@". I have not used "$@" at this time because it is not intuitively obvious that the arguments should be to the interpreter, rather than to the build module, so I don't think this should be supported unless or until a need for it determines that.) --- build-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-release.sh b/build-release.sh index 5840e4472..4eb760ddd 100755 --- a/build-release.sh +++ b/build-release.sh @@ -6,7 +6,7 @@ set -eEu function release_with() { - $1 -m build --sdist --wheel + "$1" -m build --sdist --wheel } if test -n "${VIRTUAL_ENV:-}"; then From 0920371905561d7a242a8be158b79d1a8408a7c4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 04:40:08 -0400 Subject: [PATCH 0713/1790] Extract suggest_venv out of the else block I think this is easier to read, but this is admittedly subjective. This commit also makes the separate change of adjusting comment spacing for consistency within the script. (Two spaces before "#" is not widely regarded as better than one in shell scripting, so unlike Python where PEP-8 recommends that, it would be equally good to have changed all the other places in the shell scrips to have just one.) --- build-release.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build-release.sh b/build-release.sh index 4eb760ddd..4fd4a2251 100755 --- a/build-release.sh +++ b/build-release.sh @@ -9,6 +9,11 @@ function release_with() { "$1" -m build --sdist --wheel } +function suggest_venv() { + local venv_cmd='python -m venv env && source env/bin/activate' + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" +} + if test -n "${VIRTUAL_ENV:-}"; then deps=(build twine) # Install twine along with build, as we need it later. echo "Virtual environment detected. Adding packages: ${deps[*]}" @@ -16,11 +21,7 @@ if test -n "${VIRTUAL_ENV:-}"; then echo 'Starting the build.' release_with python else - function suggest_venv() { - venv_cmd='python -m venv env && source env/bin/activate' - printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" - } trap suggest_venv ERR # This keeps the original exit (error) code. echo 'Starting the build.' - release_with python3 # Outside a venv, use python3. + release_with python3 # Outside a venv, use python3. fi From e973f527f92a932e833167c57cb4d4eeb3103429 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 05:06:59 -0400 Subject: [PATCH 0714/1790] Use some handy bash-isms in version check script The script is nonportable to other shells already because of its use of trap (and other features, such as implicit assumptions made about echo, and the function keyword), which its hashbang already clearly conveys. This uses: - $( Date: Wed, 27 Sep 2023 05:23:27 -0400 Subject: [PATCH 0715/1790] Have init script treat master unambiguously as a branch Because users may have an old version of git without "git switch", init-tests-after-clone.sh should continue to use "git checkout" to attempt to switch to master. But without "--", this suffers from the problem that it's ambiguous if master is a branch (so checkout behaves like switch) or a path (so checkout behaves like restore). There are two cases where this ambiguity can be a problem. The most common is on a fork with no master branch but also, fortunately, no file or directory named "master". Then the problem is just the error message (printed just before the script proceeds to redo the checkout with -b): error: pathspec 'master' did not match any file(s) known to git The real cause of the error is the branch being absent, as happens when a fork copies only the main branch and the upstream remote is not also set up. Adding the "--" improves the error message: fatal: invalid reference: master However, it is possible, though unlikely, for a file or directory called "master" to exist. In that case, if there is also no master branch, git discards unstaged changes made to the file or (much worse!) everywhere in that directory, potentially losing work. This commit adds "--" to the right of "master" so git never regards it as a path. This is not needed with -b, which is always followed by a symbolic name, so I have not added it there. (Note that the command is still imperfect because, for example, in rare cases there could be a master *tag*--and no master branch--in which case, as before, HEAD would be detached there and the script would attempt to continue.) --- init-tests-after-clone.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 95ced98b7..52c0c06aa 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -10,7 +10,7 @@ if [[ -z "$TRAVIS" ]]; then fi git tag __testing_point__ -git checkout master || git checkout -b master +git checkout master -- || git checkout -b master git reset --hard HEAD~1 git reset --hard HEAD~1 git reset --hard HEAD~1 From e604b469a1bdfb83f03d4c2ef1f79b45b8a5c3fa Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 05:44:02 -0400 Subject: [PATCH 0716/1790] Use 4-space indentation in all shell scripts This had been done everywhere except in init-tests-after-clone.sh. --- init-tests-after-clone.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 52c0c06aa..9d65570da 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -3,10 +3,10 @@ set -e if [[ -z "$TRAVIS" ]]; then - read -rp "This operation will destroy locally modified files. Continue ? [N/y]: " answer - if [[ ! $answer =~ [yY] ]]; then - exit 2 - fi + read -rp "This operation will destroy locally modified files. Continue ? [N/y]: " answer + if [[ ! $answer =~ [yY] ]]; then + exit 2 + fi fi git tag __testing_point__ From 19dfbd8ce4df9bde9b36eda12304ec4db71b084a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 07:33:56 -0400 Subject: [PATCH 0717/1790] Make the init script a portable POSIX shell script This translates init-tests-after-clone.sh from bash to POSIX sh, changing the hashbang and replacing all bash-specific features, so that it can be run on any POSIX-compatible ("Bourne style") shell, including but not limited to bash. This allows it to be used even on systems that don't have any version of bash installed anywhere. Only that script is modified. The other shell scripts make greater use of (and gain greater benefit from) bash features, and are also only used for building and releasing. In contrast, this script is meant to be used by most users who clone the repository. --- init-tests-after-clone.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 9d65570da..19ff0fd28 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -1,12 +1,16 @@ -#!/usr/bin/env bash +#!/bin/sh set -e -if [[ -z "$TRAVIS" ]]; then - read -rp "This operation will destroy locally modified files. Continue ? [N/y]: " answer - if [[ ! $answer =~ [yY] ]]; then - exit 2 - fi +if test -z "$TRAVIS"; then + printf 'This operation will destroy locally modified files. Continue ? [N/y]: ' >&2 + read -r answer + case "$answer" in + [yY]) + ;; + *) + exit 2 ;; + esac fi git tag __testing_point__ From 7110bf8e04f96ffb20518ebf48803a50d1e3bf22 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 14:56:20 -0400 Subject: [PATCH 0718/1790] Move extra tag-fetching step into init script This makes three related changes: - Removes "git fetch --tags" from the instructions in the readme, because the goal of this command can be achieved in the init-tests-after-clone.sh script, and because this fetch command, as written, is probably only achieving that goal in narrow cases. In clones and fetches, tags on branches are fetched by default, and the tags needed by the tests are on branches. So the situations where "git fetch --tags" was helping were (a) when the remote recently gained the tags, and (b) when the original remote was cloned in an unusual way, not fetching all tags. In both cases, the "--tags" option is not what makes that fetch get the needed tags. - Adds "git fetch --all --tags" to init-tests-after-clone.sh. The "--all" option causes it to fetch from all remotes, and this is more significant than "--tags", since the tags needed for testing are on fetched branches. This achieves what "git fetch --tags" was achieving, and it also has the benefit of getting tags from remotes that have been added but not fetched from, as happens with an upstream remote that was manually added with no further action. (It also gets branches from those remotes, but if master is on multiple remotes but at different commits then "git checkout master" may not be able to pick one. So do this *after* rather than before that.) - Skips this extra fetch, and also the submodule cloning/updating step, when running on CI. CI jobs will already have taken care of cloning the repo with submodules recursively, and fetching all available tags. In forks without tags, the necessary tags for the test are not fetched--but the upstream remote is not set up on CI, so they wouldn't be obtained anyway, not even by refetching with "--all". --- README.md | 7 +++---- init-tests-after-clone.sh | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 65c1e7bae..c17340f63 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ To clone the [the GitHub repository](https://github.com/gitpython-developers/Git ```bash git clone https://github.com/gitpython-developers/GitPython cd GitPython -git fetch --tags ./init-tests-after-clone.sh ``` @@ -114,9 +113,9 @@ See [Issue #525](https://github.com/gitpython-developers/GitPython/issues/525). ### RUNNING TESTS -_Important_: Right after cloning this repository, please be sure to have -executed `git fetch --tags` followed by the `./init-tests-after-clone.sh` -script in the repository root. Otherwise you will encounter test failures. +_Important_: Right after cloning this repository, please be sure to have executed +the `./init-tests-after-clone.sh` script in the repository root. Otherwise +you will encounter test failures. On _Windows_, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` exists in `Git\mingw64\libexec\git-core\`. diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 19ff0fd28..efa30a2dc 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -19,4 +19,8 @@ git reset --hard HEAD~1 git reset --hard HEAD~1 git reset --hard HEAD~1 git reset --hard __testing_point__ + +test -z "$TRAVIS" || exit 0 # CI jobs will already have taken care of the rest. + +git fetch --all --tags git submodule update --init --recursive From c7cdaf48865f4483bfd34e1f7527ab3e1d205dad Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 18:27:44 -0400 Subject: [PATCH 0719/1790] Reduce code duplication in version check script This extracts duplicated code to functions in check-version.sh. One effect is unfortunately that the specific commands being run are less explicitly clear when reading the script. However, small future changes, if accidentally made to one but not the other in either pair of "git status" commands, would create a much more confusing situation. So I think this change is justified overall. --- check-version.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/check-version.sh b/check-version.sh index d4200bd20..b47482d7e 100755 --- a/check-version.sh +++ b/check-version.sh @@ -10,6 +10,11 @@ trap 'echo "$0: Check failed. Stopping." >&2' ERR readonly version_path='VERSION' readonly changes_path='doc/source/changes.rst' +function check_status() { + git status -s "$@" + test -z "$(git status -s "$@")" +} + function get_latest_tag() { local config_opts printf -v config_opts ' -c versionsort.suffix=-%s' alpha beta pre rc RC @@ -23,13 +28,11 @@ test "$(cd -- "$(dirname -- "$0")" && pwd)" = "$(pwd)" # Ugly, but portable. echo "Checking that $version_path and $changes_path exist and have no uncommitted changes." test -f "$version_path" test -f "$changes_path" -git status -s -- "$version_path" "$changes_path" -test -z "$(git status -s -- "$version_path" "$changes_path")" +check_status -- "$version_path" "$changes_path" # This section can be commented out, if absolutely necessary. echo 'Checking that ALL changes are committed.' -git status -s --ignore-submodules -test -z "$(git status -s --ignore-submodules)" +check_status --ignore-submodules version_version="$(<"$version_path")" changes_version="$(awk '/^[0-9]/ {print $0; exit}' "$changes_path")" From f6dbba2ae4ad9eb3c470b30acce280a4fb6d0445 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 18:37:26 -0400 Subject: [PATCH 0720/1790] A couple more script tweaks for clarity - Because the substitition string after the hyphen is empty, "${VIRTUAL_ENV:-}" and "${VIRTUAL_ENV-}" have the same effect. However, the latter, which this changes it to, expresses the correct idea that the special case being handled is when the variable is unset: in this case, we expand an empty field rather than triggering an error due to set -u. When the variable is set but empty, it already expands to the substitution value, and including that in the special case with ":" is thus misleading. - Continuing in the vein of d18d90a (and 1e0b3f9), this removes another explicit newline by adding another echo command to print the leading blank line before the table. --- build-release.sh | 2 +- check-version.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build-release.sh b/build-release.sh index 4fd4a2251..49c13b93a 100755 --- a/build-release.sh +++ b/build-release.sh @@ -14,7 +14,7 @@ function suggest_venv() { printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" } -if test -n "${VIRTUAL_ENV:-}"; then +if test -n "${VIRTUAL_ENV-}"; then deps=(build twine) # Install twine along with build, as we need it later. echo "Virtual environment detected. Adding packages: ${deps[*]}" pip install --quiet --upgrade "${deps[@]}" diff --git a/check-version.sh b/check-version.sh index b47482d7e..dac386e46 100755 --- a/check-version.sh +++ b/check-version.sh @@ -41,7 +41,8 @@ head_sha="$(git rev-parse HEAD)" latest_tag_sha="$(git rev-parse "${latest_tag}^{commit}")" # Display a table of all the current version, tag, and HEAD commit information. -echo $'\nThe VERSION must be the same in all locations, and so must the HEAD and tag SHA' +echo +echo 'The VERSION must be the same in all locations, and so must the HEAD and tag SHA' printf '%-14s = %s\n' 'VERSION file' "$version_version" \ 'changes.rst' "$changes_version" \ 'Latest tag' "$latest_tag" \ From 5060c9dc42677c04af1b696e77228d17dac645a4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Sep 2023 19:05:25 -0400 Subject: [PATCH 0721/1790] Explain what each step in the init script achieves This adds comments to init-tests-after-clone.sh to explain what each of the steps is doing that is needed by some of the tests. It also refactors some recently added logic, in a way that lends itself to greater clarity when combined with these comments. --- init-tests-after-clone.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index efa30a2dc..a53acbbef 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -13,14 +13,26 @@ if test -z "$TRAVIS"; then esac fi +# Stop if we have run this. (You can delete __testing_point__ to let it rerun.) +# This also keeps track of where we are, so we can get back here. git tag __testing_point__ + +# The tests need a branch called master. git checkout master -- || git checkout -b master + +# The tests need a reflog history on the master branch. git reset --hard HEAD~1 git reset --hard HEAD~1 git reset --hard HEAD~1 + +# Point the master branch where we started, so we test the correct code. git reset --hard __testing_point__ -test -z "$TRAVIS" || exit 0 # CI jobs will already have taken care of the rest. +# Do some setup that CI takes care of but that may not have been done locally. +if test -z "$TRAVIS"; then + # The tests needs some version tags. Try to get them even in forks. + git fetch --all --tags -git fetch --all --tags -git submodule update --init --recursive + # The tests need submodules, including a submodule with a submodule. + git submodule update --init --recursive +fi From d5479b206fd3a5815bad618d269ecb5e1577feb8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 01:40:33 -0400 Subject: [PATCH 0722/1790] Use set -u in init script This is already done in the other shell scripts. Although init-tests-after-clone.sh does not have as many places where a bug could slip through by an inadvertently nonexistent parameter, it does have $answer (and it may have more expansions in the future). --- init-tests-after-clone.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index a53acbbef..5ca767654 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -1,8 +1,8 @@ #!/bin/sh -set -e +set -eu -if test -z "$TRAVIS"; then +if test -z "${TRAVIS-}"; then printf 'This operation will destroy locally modified files. Continue ? [N/y]: ' >&2 read -r answer case "$answer" in @@ -29,7 +29,7 @@ git reset --hard HEAD~1 git reset --hard __testing_point__ # Do some setup that CI takes care of but that may not have been done locally. -if test -z "$TRAVIS"; then +if test -z "${TRAVIS-}"; then # The tests needs some version tags. Try to get them even in forks. git fetch --all --tags From 52f9a68d04233c3be9653a4a8b56a58afb9d7093 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 03:01:43 -0400 Subject: [PATCH 0723/1790] Make the "all" Makefile target more robust This is a minor improvement to the robustness of the command for "make all", in the face of plausible future target names. - Use [[:alpha:]] instead of [a-z] because, in POSIX BRE and ERE, whether [a-z] includes capital letters depends on the current collation order. (The goal here is greater consistency across systems, and for this it would also be okay to use [[:lower:]].) - Pass -x to the command that filters out the all target itself, so that the entire name must be "all", because a future target may contain the substring "all" as part of a larger word. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 38090244c..fe9d04a18 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: all clean release force_release all: - @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + @grep -E '^[[:alpha:]].*:' Makefile | cut -d: -f1 | grep -vxF all clean: rm -rf build/ dist/ .eggs/ .tox/ From b88d07e47667194e0668431e2a871926eb54d948 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 03:16:33 -0400 Subject: [PATCH 0724/1790] Use a single awk instead of two greps and a cut This seems simpler to me, but I admit it is subjective. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fe9d04a18..d4f9acf87 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: all clean release force_release all: - @grep -E '^[[:alpha:]].*:' Makefile | cut -d: -f1 | grep -vxF all + @awk -F: '/^[[:alpha:]].*:/ && !/^all:/ {print $$1}' Makefile clean: rm -rf build/ dist/ .eggs/ .tox/ From d36818cf59d398e50cc6568f2239d69cd904883d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 03:33:59 -0400 Subject: [PATCH 0725/1790] Add a black check to pre-commit - Add the psf/black-pre-commit-mirror pre-commit hook but have it just check and report violations rather than fixing them automatically (to avoid inconsistency with all the other hooks, and also so that the linting instructions and CI lint workflow can remain the same and automatically do the black check). - Remove the "black" environment from tox.ini, since it is now part of the linting done in the "lint" environment. (But README.md remains the same, as it documented actually auto-formatting with black, which is still done the same way.) - Add missing "exclude" keys for the shellcheck and black pre-commit hooks to explicitly avoid traversing into the submodules. --- .pre-commit-config.yaml | 8 ++++++++ tox.ini | 7 +------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bacc90913..d6b496a31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,11 @@ repos: + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.9.1 + hooks: + - id: black + args: [--check, --diff] + exclude: ^git/ext/ + - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: @@ -14,6 +21,7 @@ repos: hooks: - id: shellcheck args: [--color] + exclude: ^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 diff --git a/tox.ini b/tox.ini index 82a41e22c..ed7896b59 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -env_list = py{37,38,39,310,311,312}, lint, mypy, black +env_list = py{37,38,39,310,311,312}, lint, mypy [testenv] description = Run unit tests @@ -20,11 +20,6 @@ base_python = py39 commands = mypy -p git ignore_outcome = true -[testenv:black] -description = Check style with black -base_python = py39 -commands = black --check --diff . - # Run "tox -e html" for this. It is deliberately excluded from env_list, as # unlike the other environments, this one writes outside the .tox/ directory. [testenv:html] From 4ba5ad107c4e91f62dacee9bfef277f2674fd90f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 04:27:07 -0400 Subject: [PATCH 0726/1790] Fix typo in comment --- init-tests-after-clone.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 5ca767654..5d1c16f0a 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -30,7 +30,7 @@ git reset --hard __testing_point__ # Do some setup that CI takes care of but that may not have been done locally. if test -z "${TRAVIS-}"; then - # The tests needs some version tags. Try to get them even in forks. + # The tests need some version tags. Try to get them even in forks. git fetch --all --tags # The tests need submodules, including a submodule with a submodule. From 5d8ddd9009ec38ecafddb55acfe7ad9919b1bb0d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 05:12:24 -0400 Subject: [PATCH 0727/1790] Use two hooks for black: to check, and format This splits the pre-commit hook for black into two hooks, both using the same repo and id but separately aliased: - black-check, which checks but does not modify files. This only runs when the manual stage is specified, and it is used by tox and on CI, with tox.ini and lint.yml modified accordingly. - black-format, which autoformats code. This provides the behavior most users will expect from a pre-commit hook for black. It runs automatically along with other hooks. But tox.ini and lint.yml, in addition to enabling the black-check hook, also explicitly skip the black-format hook. The effect is that in ordinary development the pre-commit hook for black does auto-formatting, but that pre-commit continues to be used effectively for running checks in which code should not be changed. --- .github/workflows/lint.yml | 4 ++++ .pre-commit-config.yaml | 8 ++++++++ README.md | 12 ++++++------ tox.ini | 4 +++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2204bb792..71e0a8853 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,3 +14,7 @@ jobs: python-version: "3.x" - uses: pre-commit/action@v3.0.0 + with: + extra_args: --hook-stage manual + env: + SKIP: black-format diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6b496a31..5664fe980 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,8 +3,16 @@ repos: rev: 23.9.1 hooks: - id: black + alias: black-check + name: black (check) args: [--check, --diff] exclude: ^git/ext/ + stages: [manual] + + - id: black + alias: black-format + name: black (format) + exclude: ^git/ext/ - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 diff --git a/README.md b/README.md index c17340f63..8cb6c88c2 100644 --- a/README.md +++ b/README.md @@ -142,22 +142,22 @@ To test, run: pytest ``` -To lint, run: +To lint, and apply automatic code formatting, run: ```bash pre-commit run --all-files ``` -To typecheck, run: +Code formatting can also be done by itself by running: -```bash -mypy -p git +``` +black . ``` -For automatic code formatting, run: +To typecheck, run: ```bash -black . +mypy -p git ``` Configuration for flake8 is in the `./.flake8` file. diff --git a/tox.ini b/tox.ini index ed7896b59..518577183 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,9 @@ commands = pytest --color=yes {posargs} [testenv:lint] description = Lint via pre-commit base_python = py39 -commands = pre-commit run --all-files +set_env = + SKIP = black-format +commands = pre-commit run --all-files --hook-stage manual [testenv:mypy] description = Typecheck with mypy From a872d9c9c90b2a99c0b1a29e10aaecbe2fa387ed Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 15:29:10 -0400 Subject: [PATCH 0728/1790] Pass --all-files explicitly so it is retained In the lint workflow, passing extra-args replaced --all-files, rather than keeping it but adding the extra arguments after it. (But --show-diff-on-failure and --color=always were still passed.) --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 71e0a8853..91dd919e0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,6 +15,6 @@ jobs: - uses: pre-commit/action@v3.0.0 with: - extra_args: --hook-stage manual + extra_args: --all-files --hook-stage manual env: SKIP: black-format From 9b9de1133b85fd308c8795a4f312d3cfbf40b75f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 15:34:28 -0400 Subject: [PATCH 0729/1790] Fix the formatting Now that the changes to lint.yml are fixed up, tested, and shown to be capable of revealing formatting errors, the formatting error I was using for testing (which is from 9fa1cee where I had introduced it inadvertently) can be fixed. --- test/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_git.py b/test/test_git.py index 1ee7b3642..cf82d9ac7 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -43,7 +43,7 @@ def tearDown(self): def _assert_logged_for_popen(self, log_watcher, name, value): re_name = re.escape(name) re_value = re.escape(str(value)) - re_line = re.compile(fr"DEBUG:git.cmd:Popen\(.*\b{re_name}={re_value}[,)]") + re_line = re.compile(rf"DEBUG:git.cmd:Popen\(.*\b{re_name}={re_value}[,)]") match_attempts = [re_line.match(message) for message in log_watcher.output] self.assertTrue(any(match_attempts), repr(log_watcher.output)) From 5d1506352cdff5f3207c5942d82cdc628cb03f3c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 05:29:50 -0400 Subject: [PATCH 0730/1790] Add "make lint" to lint without auto-formatting As on CI and with tox (but not having to build and create and use a tox environment). This may not be the best way to do it, since currently the project's makefiles are otherwise used only for things directly related to building and publishing. However, this seemed like a readily available way to run the moderately complex command that produces the same behavior as on CI or with tox, but outside a tox environment. It may be possible to improve on this in the future. --- Makefile | 5 ++++- README.md | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index d4f9acf87..839dc9f78 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -.PHONY: all clean release force_release +.PHONY: all lint clean release force_release all: @awk -F: '/^[[:alpha:]].*:/ && !/^all:/ {print $$1}' Makefile +lint: + SKIP=black-format pre-commit run --all-files --hook-stage manual + clean: rm -rf build/ dist/ .eggs/ .tox/ diff --git a/README.md b/README.md index 8cb6c88c2..5ae43dc46 100644 --- a/README.md +++ b/README.md @@ -148,11 +148,8 @@ To lint, and apply automatic code formatting, run: pre-commit run --all-files ``` -Code formatting can also be done by itself by running: - -``` -black . -``` +- Linting without modifying code can be done with: `make lint` +- Auto-formatting without other lint checks can be done with: `black .` To typecheck, run: From 6de86a85e1d17e32cab8996aa66f10783e6beced Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 05:50:35 -0400 Subject: [PATCH 0731/1790] Update readme about most of the test/lint tools Including tox. --- README.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5ae43dc46..aac69e0df 100644 --- a/README.md +++ b/README.md @@ -157,12 +157,26 @@ To typecheck, run: mypy -p git ``` -Configuration for flake8 is in the `./.flake8` file. +#### CI (and tox) -Configurations for `mypy`, `pytest`, `coverage.py`, and `black` are in `./pyproject.toml`. +The same linting, and running tests on all the different supported Python versions, will be performed: -The same linting and testing will also be performed against different supported python versions -upon submitting a pull request (or on each push if you have a fork with a "main" branch and actions enabled). +- Upon submitting a pull request. +- On each push, *if* you have a fork with a "main" branch and GitHub Actions enabled. +- Locally, if you run [`tox`](https://tox.wiki/) (this skips any Python versions you don't have installed). + +#### Configuration files + +Specific tools: + +- Configurations for `mypy`, `pytest`, `coverage.py`, and `black` are in `./pyproject.toml`. +- Configuration for `flake8` is in the `./.flake8` file. + +Orchestration tools: + +- Configuration for `pre-commit` is in the `./.pre-commit-config.yaml` file. +- Configuration for `tox` is in `./tox.ini`. +- Configuration for GitHub Actions (CI) is in files inside `./.github/workflows/`. ### Contributions From f0949096f4a2dec466bce48d46a4bf2dd598d36f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 06:31:05 -0400 Subject: [PATCH 0732/1790] Add BUILDDIR var to doc/Makefile; have tox use it This adds BUILDDIR as a variable in the documentation generation makefile that, along SPHINXOPTS, SPHINXBUILD, and PAPER, defaults to the usual best value but can be set when invoking make. The main use for choosing a different build output directory is to test building without overwriting or otherwise interfering with any files from a build one may really want to use. tox.ini is thus modified to pass a path pointing inside its temporary directory as BUILDDIR, so the "html" tox environment now makes no changes outside the .tox directory. This is thus suitable for being run automatically, so the "html" environment is now in the envlist. --- doc/Makefile | 43 ++++++++++++++++++++++--------------------- tox.ini | 8 ++++---- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index ef2d60e5f..ddeadbd7e 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,6 +2,7 @@ # # You can set these variables from the command line. +BUILDDIR = build SPHINXOPTS = -W SPHINXBUILD = sphinx-build PAPER = @@ -9,7 +10,7 @@ PAPER = # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html web pickle htmlhelp latex changes linkcheck @@ -24,52 +25,52 @@ help: @echo " linkcheck to check all external links for integrity" clean: - -rm -rf build/* + -rm -rf $(BUILDDIR)/* html: - mkdir -p build/html build/doctrees - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html + mkdir -p $(BUILDDIR)/html $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo - @echo "Build finished. The HTML pages are in build/html." + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." pickle: - mkdir -p build/pickle build/doctrees - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) build/pickle + mkdir -p $(BUILDDIR)/pickle $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." web: pickle json: - mkdir -p build/json build/doctrees - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) build/json + mkdir -p $(BUILDDIR)/json $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - mkdir -p build/htmlhelp build/doctrees - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp + mkdir -p $(BUILDDIR)/htmlhelp $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in build/htmlhelp." + ".hhp project file in $(BUILDDIR)/htmlhelp." latex: - mkdir -p build/latex build/doctrees - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex + mkdir -p $(BUILDDIR)/latex $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo - @echo "Build finished; the LaTeX files are in build/latex." + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." changes: - mkdir -p build/changes build/doctrees - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes + mkdir -p $(BUILDDIR)/changes $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo - @echo "The overview file is in build/changes." + @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - mkdir -p build/linkcheck build/doctrees - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck + mkdir -p $(BUILDDIR)/linkcheck $(BUILDDIR)/doctrees + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ - "or in build/linkcheck/output.txt." + "or in $(BUILDDIR)/linkcheck/output.txt." diff --git a/tox.ini b/tox.ini index 518577183..47a7a6209 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -env_list = py{37,38,39,310,311,312}, lint, mypy +env_list = py{37,38,39,310,311,312}, lint, mypy, html [testenv] description = Run unit tests @@ -22,11 +22,11 @@ base_python = py39 commands = mypy -p git ignore_outcome = true -# Run "tox -e html" for this. It is deliberately excluded from env_list, as -# unlike the other environments, this one writes outside the .tox/ directory. [testenv:html] description = Build HTML documentation base_python = py39 deps = -r doc/requirements.txt allowlist_externals = make -commands = make -C doc html +commands = + make BUILDDIR={env_tmp_dir}/doc/build -C doc clean + make BUILDDIR={env_tmp_dir}/doc/build -C doc html From fc969807086d4483c4c32b80d2c2b67a6c6813e7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 15:27:11 -0400 Subject: [PATCH 0733/1790] Have init script check for GitHub Actions As well as still checking for Travis, for backward compatibility and because experience shows that this is safe. The check can be much broader, and would be more accurate, with fewer false negatives. But a false positive can result in local data loss because the script does hard resets on CI without prompting for confirmation. So for now, this just checks $TRAVIS and $GITHUB_ACTIONS. Now that GHA is included, the CI workflows no longer need to set $TRAVIS when running the script, so that is removed. --- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- init-tests-after-clone.sh | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index e818803f1..cd913385f 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -40,7 +40,7 @@ jobs: - name: Prepare this repo for tests run: | - TRAVIS=yes ./init-tests-after-clone.sh + ./init-tests-after-clone.sh - name: Set git user identity and command aliases for the tests run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e43317807..2a82e0e03 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -37,7 +37,7 @@ jobs: - name: Prepare this repo for tests run: | - TRAVIS=yes ./init-tests-after-clone.sh + ./init-tests-after-clone.sh - name: Set git user identity and command aliases for the tests run: | diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 5d1c16f0a..4697c2ecc 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -2,7 +2,12 @@ set -eu -if test -z "${TRAVIS-}"; then +ci() { + # For now, check just these, as a false positive could lead to data loss. + test -n "${TRAVIS-}" || test -n "${GITHUB_ACTIONS-}" +} + +if ! ci; then printf 'This operation will destroy locally modified files. Continue ? [N/y]: ' >&2 read -r answer case "$answer" in @@ -29,7 +34,7 @@ git reset --hard HEAD~1 git reset --hard __testing_point__ # Do some setup that CI takes care of but that may not have been done locally. -if test -z "${TRAVIS-}"; then +if ! ci; then # The tests need some version tags. Try to get them even in forks. git fetch --all --tags From b98f15e46a7d5f343b1303b55bc4dae2d18bd621 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Sep 2023 23:59:14 -0400 Subject: [PATCH 0734/1790] Get tags for tests from original repo as fallback This extends the init script so it tries harder to get version tags: - As before, locally, "git fetch --all --tags" is always run. (This also fetches other objects, and the developer experience would be undesirably inconsistent if that were only sometimes done.) - On CI, run it if version tags are absent. The criterion followed here and in subsequent steps is that if any existing tag starts with a digit, or with the letter v followed by a digit, we regard version tags to be present. This is to balance the benefit of getting the tags (to make the tests work) against the risk of creating a very confusing situation in clones of forks that publish packages or otherwise use their own separate versioning scheme (especially if those tags later ended up being pushed). - Both locally and on CI, after that, if version tags are absent, try to copy them from the original gitpython-developers/GitPython repo, without copying other tags or adding that repo as a remote. Copy only tags that start with a digit, since v+digit tags aren't currently used in this project (though forks may use them). This is a fallback option and it always displays a warning. If it fails, another message is issued for that. Unexpected failure to access the repo terminates the script with a failing exit status, but the absence of version tags in the fallback remote does not, so CI jobs can continue and reveal which tests fail as a result. On GitHub Actions CI specifically, the Actions syntax for creating a warning annotation on the workflow is used, but the warning is still also written to stderr (as otherwise). GHA workflow annotations don't support multi-line warnings, so the message is adjusted into a single line where needed. --- init-tests-after-clone.sh | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 4697c2ecc..5e31e3315 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -2,11 +2,25 @@ set -eu +fallback_repo_for_tags='https://github.com/gitpython-developers/GitPython.git' + ci() { # For now, check just these, as a false positive could lead to data loss. test -n "${TRAVIS-}" || test -n "${GITHUB_ACTIONS-}" } +no_version_tags() { + test -z "$(git tag -l '[0-9]*' 'v[0-9]*')" +} + +warn() { + printf '%s\n' "$@" >&2 # Warn in step output. + + if test -n "${GITHUB_ACTIONS-}"; then + printf '::warning ::%s\n' "$*" >&2 # Annotate workflow. + fi +} + if ! ci; then printf 'This operation will destroy locally modified files. Continue ? [N/y]: ' >&2 read -r answer @@ -33,11 +47,27 @@ git reset --hard HEAD~1 # Point the master branch where we started, so we test the correct code. git reset --hard __testing_point__ -# Do some setup that CI takes care of but that may not have been done locally. +# The tests need submodules. (On CI, they would already have been checked out.) if ! ci; then - # The tests need some version tags. Try to get them even in forks. + git submodule update --init --recursive +fi + +# The tests need some version tags. Try to get them even in forks. This fetch +# gets other objects too, so for a consistent experience, always do it locally. +if ! ci || no_version_tags; then git fetch --all --tags +fi - # The tests need submodules, including a submodule with a submodule. - git submodule update --init --recursive +# If we still have no version tags, try to get them from the original repo. +if no_version_tags; then + warn 'No local or remote version tags found. Trying fallback remote:' \ + "$fallback_repo_for_tags" + + # git fetch supports * but not [], and --no-tags means no *other* tags, so... + printf 'refs/tags/%d*:refs/tags/%d*\n' 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7 8 8 9 9 | + xargs git fetch --no-tags "$fallback_repo_for_tags" + + if no_version_tags; then + warn 'No version tags found anywhere. Some tests will fail.' + fi fi From 7cca7d2245a504047188943623cc58c4c2e0c35f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Sep 2023 10:28:17 -0400 Subject: [PATCH 0735/1790] Don't print the exact same warning twice In the step output, the warning that produces a workflow annotation is fully readable (and even made to stand out), so it doesn't need to be printed in the plain way as well, which can be reserved for when GitHub Actions is not detected. --- init-tests-after-clone.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 5e31e3315..49df49daa 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -14,10 +14,10 @@ no_version_tags() { } warn() { - printf '%s\n' "$@" >&2 # Warn in step output. - if test -n "${GITHUB_ACTIONS-}"; then printf '::warning ::%s\n' "$*" >&2 # Annotate workflow. + else + printf '%s\n' "$@" >&2 fi } From e4e009d03b890d456b4c1ff2411591d3a50dcdd0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Sep 2023 16:59:58 -0400 Subject: [PATCH 0736/1790] Reword comment to fix ambiguity --- init-tests-after-clone.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 49df49daa..21d1f86d8 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -52,8 +52,8 @@ if ! ci; then git submodule update --init --recursive fi -# The tests need some version tags. Try to get them even in forks. This fetch -# gets other objects too, so for a consistent experience, always do it locally. +# The tests need some version tags. Try to get them even in forks. This fetches +# other objects too. So, locally, we always do it, for a consistent experience. if ! ci || no_version_tags; then git fetch --all --tags fi From e16e4c0099cd0197b5c80ce6ec9a6b4bca41845e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Sep 2023 18:58:08 -0400 Subject: [PATCH 0737/1790] Format all YAML files in the same style --- .github/dependabot.yml | 2 +- .pre-commit-config.yaml | 68 ++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 203f3c889..8c139c7be 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,4 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "weekly" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5664fe980..be97d5f9b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,39 +1,39 @@ repos: - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 - hooks: - - id: black - alias: black-check - name: black (check) - args: [--check, --diff] - exclude: ^git/ext/ - stages: [manual] +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.9.1 + hooks: + - id: black + alias: black-check + name: black (check) + args: [--check, --diff] + exclude: ^git/ext/ + stages: [manual] - - id: black - alias: black-format - name: black (format) - exclude: ^git/ext/ + - id: black + alias: black-format + name: black (format) + exclude: ^git/ext/ - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==23.9.16 - - flake8-comprehensions==3.14.0 - - flake8-typing-imports==1.14.0 - exclude: ^doc|^git/ext/ +- repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear==23.9.16 + - flake8-comprehensions==3.14.0 + - flake8-typing-imports==1.14.0 + exclude: ^doc|^git/ext/ - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.5 - hooks: - - id: shellcheck - args: [--color] - exclude: ^git/ext/ +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.5 + hooks: + - id: shellcheck + args: [--color] + exclude: ^git/ext/ - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-toml - - id: check-yaml - - id: check-merge-conflict +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-toml + - id: check-yaml + - id: check-merge-conflict From 62c024e277820b2fd3764a0499a71f03d4aa432d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Sep 2023 19:33:07 -0400 Subject: [PATCH 0738/1790] Let tox run lint, mypy, and html envs without 3.9 The tox environments that are not duplicated per Python version were set to run on Python 3.9, for consistency with Cygwin, where 3.9 is the latest available (through official Cygwin packages), so output can be compared between Cygwin and other platforms while troubleshooting problems. However, this prevented them from running altogether on systems where Python 3.9 was not installed. So I've added all the other Python versions the project supports as fallback versions for those tox environments to use, in one of the reasonable precedence orders, while keeping 3.9 as the first choice for the same reasons as before. --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 47a7a6209..f9ac25b78 100644 --- a/tox.ini +++ b/tox.ini @@ -11,20 +11,20 @@ commands = pytest --color=yes {posargs} [testenv:lint] description = Lint via pre-commit -base_python = py39 +base_python = py{39,310,311,312,38,37} set_env = SKIP = black-format commands = pre-commit run --all-files --hook-stage manual [testenv:mypy] description = Typecheck with mypy -base_python = py39 +base_python = py{39,310,311,312,38,37} commands = mypy -p git ignore_outcome = true [testenv:html] description = Build HTML documentation -base_python = py39 +base_python = py{39,310,311,312,38,37} deps = -r doc/requirements.txt allowlist_externals = make commands = From 9e245d0561ca2f6249e9435a04761da2c64977f1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 1 Oct 2023 00:03:16 -0400 Subject: [PATCH 0739/1790] Update readme: CI jobs not just for "main" branch This changed a while ago but README.md still listed having a main branch as a condition for automatic CI linting and testing in a fork. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aac69e0df..021aab15f 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ mypy -p git The same linting, and running tests on all the different supported Python versions, will be performed: - Upon submitting a pull request. -- On each push, *if* you have a fork with a "main" branch and GitHub Actions enabled. +- On each push, *if* you have a fork with GitHub Actions enabled. - Locally, if you run [`tox`](https://tox.wiki/) (this skips any Python versions you don't have installed). #### Configuration files From c2472e9b57afde98bb18ada55ae978721a27713d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 1 Oct 2023 00:11:37 -0400 Subject: [PATCH 0740/1790] Note that the init script can be run from Git Bash This is probably the *only* way anyone should run that script on Windows, but I don't know of specific bad things that happen if it is run in some other way, such as with WSL bash, aside from messing up line endings, which users are likely to notice anyway. This commit also clarifies the instructions by breaking up another paragraph that really represented two separate steps. --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 021aab15f..69fb54c9f 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,17 @@ cd GitPython ./init-tests-after-clone.sh ``` +On Windows, `./init-tests-after-clone.sh` can be run in a Git Bash shell. + If you are cloning [your own fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks), then replace the above `git clone` command with one that gives the URL of your fork. Or use this [`gh`](https://cli.github.com/) command (assuming you have `gh` and your fork is called `GitPython`): ```bash gh repo clone GitPython ``` -Having cloned the repo, create and activate your [virtual environment](https://docs.python.org/3/tutorial/venv.html). Then make an [editable install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): +Having cloned the repo, create and activate your [virtual environment](https://docs.python.org/3/tutorial/venv.html). + +Then make an [editable install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): ```bash pip install -e ".[test]" From 8edc53b7c9be8beebef240b9a2a8fff19b79d21e Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:18:30 -0600 Subject: [PATCH 0741/1790] Clean up __all__ in main, and explicit imports in exc - explicit imports in exc added to avoid linting errors in __init__ --- git/__init__.py | 88 ++++++++++++++++++++++++++++++++++++++++++------- git/exc.py | 13 ++++++-- 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index cc0ca2136..00e7b8dd3 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -63,17 +63,83 @@ def _init_externals() -> None: # __all__ must be statically defined by py.typed support # __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] -__all__ = ['BadName', 'safe_decode', - 'remove_password_if_present', 'List', 'Sequence', 'Tuple', 'Union', 'TYPE_CHECKING', - 'PathLike', 'GitError', 'InvalidGitRepositoryError', 'WorkTreeRepositoryUnsupported', - 'NoSuchPathError', 'UnsafeProtocolError', 'UnsafeOptionError', 'CommandError', 'GitCommandNotFound', - 'GitCommandError', 'CheckoutError', 'CacheError', 'UnmergedEntriesError', 'HookExecutionError', - 'RepositoryDirtyError', 'Optional', 'GitConfigParser', 'Object', 'IndexObject', 'Blob', 'Commit', - 'Submodule', 'UpdateProgress', 'RootModule', 'RootUpdateProgress', 'TagObject', 'TreeModifier', - 'Tree', 'SymbolicReference', 'Reference', 'HEAD', 'Head', 'TagReference', 'Tag', 'RemoteReference', - 'RefLog', 'RefLogEntry', 'Diffable', 'DiffIndex', 'Diff', 'NULL_TREE', 'GitCmdObjectDB', 'GitDB', - 'Git', 'Repo', 'RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote', 'IndexFile', 'StageType', - 'BlobFilter', 'BaseIndexEntry', 'IndexEntry', 'LockFile', 'BlockingLockFile', 'Stats', 'Actor', 'rmtree'] +__all__ = [ + 'Actor', + 'AmbiguousObjectName', + 'BadName', + 'BadObject', + 'BadObjectType', + 'BaseIndexEntry', + 'Blob', + 'BlobFilter', + 'BlockingLockFile', + 'CacheError', + 'CheckoutError', + 'CommandError', + 'Commit', + 'Diff', + 'DiffIndex', + 'Diffable', + 'FetchInfo', + 'Git', + 'GitCmdObjectDB', + 'GitCommandError', + 'GitCommandNotFound', + 'GitConfigParser', + 'GitDB', + 'GitError', + 'HEAD', + 'Head', + 'HookExecutionError', + 'IndexEntry', + 'IndexFile', + 'IndexObject', + 'InvalidDBRoot', + 'InvalidGitRepositoryError', + 'List', + 'LockFile', + 'NULL_TREE', + 'NoSuchPathError', + 'ODBError', + 'Object', + 'Optional', + 'ParseError', + 'PathLike', + 'PushInfo', + 'RefLog', + 'RefLogEntry', + 'Reference', + 'Remote', + 'RemoteProgress', + 'RemoteReference', + 'Repo', + 'RepositoryDirtyError', + 'RootModule', + 'RootUpdateProgress', + 'Sequence', + 'StageType', + 'Stats', + 'Submodule', + 'SymbolicReference', + 'TYPE_CHECKING', + 'Tag', + 'TagObject', + 'TagReference', + 'Tree', + 'TreeModifier', + 'Tuple', + 'Union', + 'UnmergedEntriesError', + 'UnsafeOptionError', + 'UnsafeProtocolError', + 'UnsupportedOperation', + 'UpdateProgress', + 'WorkTreeRepositoryUnsupported', + 'remove_password_if_present', + 'rmtree', + 'safe_decode', + 'to_hex_sha', +] # { Initialize git executable path GIT_OK = None diff --git a/git/exc.py b/git/exc.py index 775528bf6..0c939f929 100644 --- a/git/exc.py +++ b/git/exc.py @@ -5,8 +5,17 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php """ Module containing all exceptions thrown throughout the git package, """ -from gitdb.exc import BadName # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 -from gitdb.exc import * # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 +from gitdb.exc import ( + AmbiguousObjectName, + BadName, + BadObject, + BadObjectType, + InvalidDBRoot, + ODBError, + ParseError, + UnsupportedOperation, + to_hex_sha, +) # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 from git.compat import safe_decode from git.util import remove_password_if_present From 70098c9c99db872ac98a6b5c7a591a42cadf049f Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:22:44 -0600 Subject: [PATCH 0742/1790] Add __all__ to exc for linting --- gitdb/exc.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gitdb/exc.py b/gitdb/exc.py index 73970372b..752dafdbb 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -5,6 +5,17 @@ """Module with common exceptions""" from gitdb.util import to_hex_sha +__all__ = [ + 'AmbiguousObjectName', + 'BadName', + 'BadObject', + 'BadObjectType', + 'InvalidDBRoot', + 'ODBError', + 'ParseError', + 'UnsupportedOperation', + 'to_hex_sha', +] class ODBError(Exception): """All errors thrown by the object database""" From 9ac6bb0b58c88490a8195a62b9b9662bdd538bd6 Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:53:18 -0600 Subject: [PATCH 0743/1790] Add @UnusedImport to recent changes --- git/exc.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/git/exc.py b/git/exc.py index 0c939f929..936381fd8 100644 --- a/git/exc.py +++ b/git/exc.py @@ -6,16 +6,16 @@ """ Module containing all exceptions thrown throughout the git package, """ from gitdb.exc import ( - AmbiguousObjectName, - BadName, - BadObject, - BadObjectType, - InvalidDBRoot, - ODBError, - ParseError, - UnsupportedOperation, - to_hex_sha, -) # NOQA @UnusedWildImport skipcq: PYL-W0401, PYL-W0614 + AmbiguousObjectName, # @UnusedImport + BadName, # @UnusedImport + BadObject, # @UnusedImport + BadObjectType, # @UnusedImport + InvalidDBRoot, # @UnusedImport + ODBError, # @UnusedImport + ParseError, # @UnusedImport + UnsupportedOperation, # @UnusedImport + to_hex_sha, # @UnusedImport +) from git.compat import safe_decode from git.util import remove_password_if_present From 04f3200723d67aaf5d46106c5c5602a0d5caae94 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Oct 2023 11:40:01 -0400 Subject: [PATCH 0744/1790] Ask git where its daemon is and use that This changes the test helpers on Windows to use "git --exec-path" (with whatever "git" GitPython is using) to find the directory that contains "git-daemon.exe", instead of finding it in a PATH search. --- README.md | 3 --- test/lib/helper.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 69fb54c9f..b9e61912a 100644 --- a/README.md +++ b/README.md @@ -121,9 +121,6 @@ _Important_: Right after cloning this repository, please be sure to have execute the `./init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. -On _Windows_, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` -exists in `Git\mingw64\libexec\git-core\`. - #### Install test dependencies Ensure testing libraries are installed. This is taken care of already if you installed with: diff --git a/test/lib/helper.py b/test/lib/helper.py index 1de904610..d415ba2e7 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -182,7 +182,7 @@ def git_daemon_launched(base_path, ip, port): # and then CANNOT DIE! # So, invoke it as a single command. daemon_cmd = [ - "git-daemon", + osp.join(Git()._call_process("--exec-path"), "git-daemon"), "--enable=receive-pack", "--listen=%s" % ip, "--port=%s" % port, From 28142630f6c3bfd83dcd654c102235c5c00075e5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 01:47:38 -0400 Subject: [PATCH 0745/1790] Add a missing PermissionError xfail on Windows One of the tests that was commented as being skipped as a result of SkipTest rasied in git.util.rmtree or one of the functions that calls it, test_git_submodule_compatibility, was not skipped in that way and was actually failing on Windows with PermissionError. It appears that the cause of failure changed over time, so that it once involved rmtree but no longer does. This removes the outdated comment and adds an xfail mark instead, specific to PermissionError and with a message identifying where in the test case's logic the PermissionError is currently triggered. --- test/test_submodule.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 79ff2c5f2..0ebd8c51d 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -819,9 +819,11 @@ def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): assert commit_sm.binsha == sm_too.binsha assert sm_too.binsha != sm.binsha - # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, ## ACTUALLY skipped by `git.submodule.base#L869`. - # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " - # "'C:\\Users\\appveyor\\AppData\\Local\\Temp\\1\\test_work_tree_unsupportedryfa60di\\master_repo\\.git\\objects\\pack\\pack-bc9e0787aef9f69e1591ef38ea0a6f566ec66fe3.idx") # noqa E501 + @pytest.mark.xfail( + HIDE_WINDOWS_KNOWN_ERRORS, + reason='"The process cannot access the file because it is being used by another process" on call to sm.move', + raises=PermissionError, + ) @with_rw_directory def test_git_submodule_compatibility(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) From fba59aa32119e22b1b300fea8959c0abd3c9f863 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 04:39:56 -0400 Subject: [PATCH 0746/1790] Update "ACTUALLY skipped by" comments The remaining "ACTUALLY skipped by" comments in the test suite were for tests that are actually skipped by SkipTest exceptions raised from the code under test. But the information provided about where in the code they were skipped was out of date, and also not detailed enough because references to line numbers become stale when code is added or removed in the referenced module before the referenced code. This updates them and also provides more detailed information about the referenced code doing the skipping. The error messages are the same as before, and the paths are the same in relevant details, so this doesn't modify those parts of the comments. --- test/test_docs.py | 5 ++++- test/test_submodule.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_docs.py b/test/test_docs.py index 79e1f1be4..b38ac31b0 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -21,7 +21,10 @@ def tearDown(self): gc.collect() - # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, ## ACTUALLY skipped by `git.submodule.base#L869`. + # ACTUALLY skipped by git.objects.submodule.base.Submodule.remove, at the last + # rmtree call (in "handle separate bare repository"), line 1082. + # + # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " # "'C:\\Users\\appveyor\\AppData\\Local\\Temp\\1\\test_work_tree_unsupportedryfa60di\\master_repo\\.git\\objects\\pack\\pack-bc9e0787aef9f69e1591ef38ea0a6f566ec66fe3.idx") # noqa E501 @with_rw_directory diff --git a/test/test_submodule.py b/test/test_submodule.py index 0ebd8c51d..1c105c816 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -457,7 +457,10 @@ def _do_base_tests(self, rwrepo): True, ) - # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, ## ACTUALLY skipped by `git.submodule.base#L869`. + # ACTUALLY skipped by git.util.rmtree (in local onerror function), called via + # git.objects.submodule.base.Submodule.remove at "method(mp)", line 1018. + # + # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: fails with: PermissionError: [WinError 32] The process cannot access the file because" # "it is being used by another process: " # "'C:\\Users\\ankostis\\AppData\\Local\\Temp\\tmp95c3z83bnon_bare_test_base_rw\\git\\ext\\gitdb\\gitdb\\ext\\smmap'") # noqa E501 From 5039df3560d321af1746bbecbeb1b2838daf7f91 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 06:31:27 -0400 Subject: [PATCH 0747/1790] Eliminate duplicate rmtree try-except logic In git.util.rmtree, exceptions are caught and conditionally (depending on the value of HIDE_WINDOWS_KNOWN_ERRORS) reraised wrapped in a unittest.SkipTest exception. Although this logic is part of git.util.rmtree itself, two of the calls to that rmtree function contain this same logic. This is not quite a refactoring: because SkipTest derives from Exception, and Exception rather than PermissionError is being caught including in the duplicated logic, duplicated logic where git.util.rmtree was called added another layer of indirection in the chained exceptions leading back to the original that was raised in an unsuccessful attempt to delete a file or directory in rmtree. However, that appeared unintended and may be considered a minor bug. The new behavior, differing only subtly, is preferable. --- git/objects/submodule/base.py | 20 ++------------------ test/test_docs.py | 4 ++-- test/test_submodule.py | 2 +- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index c7e7856f0..6fe946084 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -29,7 +29,6 @@ unbare_repo, IterableList, ) -from git.util import HIDE_WINDOWS_KNOWN_ERRORS import os.path as osp @@ -1060,28 +1059,13 @@ def remove( import gc gc.collect() - try: - rmtree(str(wtd)) - except Exception as ex: - if HIDE_WINDOWS_KNOWN_ERRORS: - from unittest import SkipTest - - raise SkipTest("FIXME: fails with: PermissionError\n {}".format(ex)) from ex - raise + rmtree(str(wtd)) # END delete tree if possible # END handle force if not dry_run and osp.isdir(git_dir): self._clear_cache() - try: - rmtree(git_dir) - except Exception as ex: - if HIDE_WINDOWS_KNOWN_ERRORS: - from unittest import SkipTest - - raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex - else: - raise + rmtree(git_dir) # end handle separate bare repository # END handle module deletion diff --git a/test/test_docs.py b/test/test_docs.py index b38ac31b0..f17538aeb 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -21,8 +21,8 @@ def tearDown(self): gc.collect() - # ACTUALLY skipped by git.objects.submodule.base.Submodule.remove, at the last - # rmtree call (in "handle separate bare repository"), line 1082. + # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last call to it via + # git.objects.submodule.base.Submodule.remove (at "handle separate bare repository"), line 1068. # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " diff --git a/test/test_submodule.py b/test/test_submodule.py index 1c105c816..318a5afde 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -458,7 +458,7 @@ def _do_base_tests(self, rwrepo): ) # ACTUALLY skipped by git.util.rmtree (in local onerror function), called via - # git.objects.submodule.base.Submodule.remove at "method(mp)", line 1018. + # git.objects.submodule.base.Submodule.remove at "method(mp)", line 1017. # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: fails with: PermissionError: [WinError 32] The process cannot access the file because" From 683a3eeba838bb786bb1f334c963deb8e13eed0f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 13:18:09 -0400 Subject: [PATCH 0748/1790] Clean up git.objects.submodule.base imports This reorders them lexicographically within each group, makes spacing/formatting more consistent, and removes the old comment about needing a dict to set .name, which had originally been on what later became the BytesIO import but had become separate from it. (In Python 2, there was a cStringIO type, which could provide a speed advantage over StringIO, but its instances, not having instance dictionaries, didn't support the dynamic creation of new attributes. This was changed to StringIO in 00ce31a to allow .name to be added. It was changed to BytesIO in bc8c912 to work with bytes on both Python 2 and Python 3. The comment about needing a dict later ended up on the preceding line in 0210e39, at which point its meaning was unclear. Because Python 2 is no longer supported and Python 3 has no cStringIO type, the comment is no longer needed, and this commit removes it.) --- git/objects/submodule/base.py | 24 +++++++++--------------- test/test_docs.py | 2 +- test/test_submodule.py | 2 +- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 6fe946084..13d897263 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,43 +1,37 @@ -# need a dict to set bloody .name field from io import BytesIO import logging import os +import os.path as osp import stat import uuid import git from git.cmd import Git -from git.compat import ( - defenc, - is_win, -) -from git.config import SectionConstraint, GitConfigParser, cp +from git.compat import defenc, is_win +from git.config import GitConfigParser, SectionConstraint, cp from git.exc import ( + BadName, InvalidGitRepositoryError, NoSuchPathError, RepositoryDirtyError, - BadName, ) from git.objects.base import IndexObject, Object from git.objects.util import TraversableIterableObj - from git.util import ( - join_path_native, - to_native_path_linux, + IterableList, RemoteProgress, + join_path_native, rmtree, + to_native_path_linux, unbare_repo, - IterableList, ) -import os.path as osp - from .util import ( + SubmoduleConfigParser, + find_first_remote_branch, mkhead, sm_name, sm_section, - SubmoduleConfigParser, - find_first_remote_branch, ) diff --git a/test/test_docs.py b/test/test_docs.py index f17538aeb..d1ed46926 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -22,7 +22,7 @@ def tearDown(self): gc.collect() # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last call to it via - # git.objects.submodule.base.Submodule.remove (at "handle separate bare repository"), line 1068. + # git.objects.submodule.base.Submodule.remove (at "handle separate bare repository"), line 1062. # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " diff --git a/test/test_submodule.py b/test/test_submodule.py index 318a5afde..31a555ce2 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -458,7 +458,7 @@ def _do_base_tests(self, rwrepo): ) # ACTUALLY skipped by git.util.rmtree (in local onerror function), called via - # git.objects.submodule.base.Submodule.remove at "method(mp)", line 1017. + # git.objects.submodule.base.Submodule.remove at "method(mp)", line 1011. # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: fails with: PermissionError: [WinError 32] The process cannot access the file because" From 2fe7f3c4a6f9870bb332761740c883a2c2ff2487 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 22:46:43 -0400 Subject: [PATCH 0749/1790] Test current expected behavior of git.util.rmtree --- test/test_util.py | 70 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 2b1e518ed..552700c98 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -5,11 +5,13 @@ # the BSD License: https://opensource.org/license/bsd-3-clause/ import os +import pathlib import pickle +import stat import sys import tempfile import time -from unittest import mock, skipUnless +from unittest import SkipTest, mock, skipIf, skipUnless from datetime import datetime import ddt @@ -19,25 +21,26 @@ from git.compat import is_win from git.objects.util import ( altz_to_utctz_str, - utctz_to_altz, - verify_utctz, + from_timestamp, parse_date, tzoffset, - from_timestamp, + utctz_to_altz, + verify_utctz, ) from test.lib import ( TestBase, with_rw_repo, ) from git.util import ( - LockFile, - BlockingLockFile, - get_user_id, Actor, + BlockingLockFile, IterableList, + LockFile, cygpath, decygpath, + get_user_id, remove_password_if_present, + rmtree, ) @@ -85,6 +88,59 @@ def setup(self): "array": [42], } + def test_rmtree_deletes_nested_dir_with_files(self): + with tempfile.TemporaryDirectory() as parent: + td = pathlib.Path(parent, "testdir") + for d in td, td / "q", td / "s": + d.mkdir() + for f in td / "p", td / "q" / "w", td / "q" / "x", td / "r", td / "s" / "y", td / "s" / "z": + f.write_bytes(b"") + + try: + rmtree(td) + except SkipTest as ex: + self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + + self.assertFalse(td.exists()) + + @skipIf(sys.platform == "cygwin", "Cygwin can't set the permissions that make the test meaningful.") + def test_rmtree_deletes_dir_with_readonly_files(self): + # Automatically works on Unix, but requires special handling on Windows. + with tempfile.TemporaryDirectory() as parent: + td = pathlib.Path(parent, "testdir") + for d in td, td / "sub": + d.mkdir() + for f in td / "x", td / "sub" / "y": + f.write_bytes(b"") + f.chmod(0) + + try: + rmtree(td) + except SkipTest as ex: + self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + + self.assertFalse(td.exists()) + + @skipIf(sys.platform == "cygwin", "Cygwin can't set the permissions that make the test meaningful.") + @skipIf(sys.version_info < (3, 8), "In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") + def test_rmtree_can_wrap_exceptions(self): + with tempfile.TemporaryDirectory() as parent: + td = pathlib.Path(parent, "testdir") + td.mkdir() + (td / "x").write_bytes(b"") + (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. + td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. + + # Access the module through sys.modules so it is unambiguous which module's + # attribute we patch: the original git.util, not git.index.util even though + # git.index.util "replaces" git.util and is what "import git.util" gives us. + with mock.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", True): + # Disable common chmod functions so the callback can't fix the problem. + with mock.patch.object(os, "chmod"), mock.patch.object(pathlib.Path, "chmod"): + # Now we can see how an intractable PermissionError is treated. + with self.assertRaises(SkipTest): + rmtree(td) + # FIXME: Mark only the /proc-prefixing cases xfail, somehow (or fix them). @pytest.mark.xfail( reason="Many return paths prefixed /proc/cygdrive instead.", From d42cd721112d748c35d0abd11ba8dfc71052e864 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Oct 2023 01:45:57 -0400 Subject: [PATCH 0750/1790] Test situations git.util.rmtree shouldn't wrap One of the new test cases fails, showing the bug where git.util.rmtree wraps any exception in SkipTest when HIDE_WINDOWS_KNOWN_ERRORS is true, even though the message it uses (and its purpose) is specific to PermissionError. The other new cases pass, because wrapping exceptions in SkipTest rightly does not occur when HIDE_WINDOWS_KNOWN_ERRORS is false. --- test/test_util.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 552700c98..a852eb975 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -4,6 +4,7 @@ # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ +import contextlib import os import pathlib import pickle @@ -121,16 +122,31 @@ def test_rmtree_deletes_dir_with_readonly_files(self): self.assertFalse(td.exists()) - @skipIf(sys.platform == "cygwin", "Cygwin can't set the permissions that make the test meaningful.") - @skipIf(sys.version_info < (3, 8), "In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") - def test_rmtree_can_wrap_exceptions(self): + @staticmethod + @contextlib.contextmanager + def _tmpdir_to_force_permission_error(): + if sys.platform == "cygwin": + raise SkipTest("Cygwin can't set the permissions that make the test meaningful.") + if sys.version_info < (3, 8): + raise SkipTest("In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") + with tempfile.TemporaryDirectory() as parent: td = pathlib.Path(parent, "testdir") td.mkdir() (td / "x").write_bytes(b"") (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. + yield td + @staticmethod + @contextlib.contextmanager + def _tmpdir_for_file_not_found(): + with tempfile.TemporaryDirectory() as parent: + yield pathlib.Path(parent, "testdir") # It is deliberately never created. + + def test_rmtree_can_wrap_exceptions(self): + """Our rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + with self._tmpdir_to_force_permission_error() as td: # Access the module through sys.modules so it is unambiguous which module's # attribute we patch: the original git.util, not git.index.util even though # git.index.util "replaces" git.util and is what "import git.util" gives us. @@ -141,6 +157,25 @@ def test_rmtree_can_wrap_exceptions(self): with self.assertRaises(SkipTest): rmtree(td) + @ddt.data( + (False, PermissionError, _tmpdir_to_force_permission_error), + (False, FileNotFoundError, _tmpdir_for_file_not_found), + (True, FileNotFoundError, _tmpdir_for_file_not_found), + ) + def test_rmtree_does_not_wrap_unless_called_for(self, case): + """Our rmtree doesn't wrap non-PermissionError, nor when HIDE_WINDOWS_KNOWN_ERRORS is false.""" + hide_windows_known_errors, exception_type, tmpdir_context_factory = case + + with tmpdir_context_factory() as td: + # See comments in test_rmtree_can_wrap_exceptions regarding the patching done here. + with mock.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors): + with mock.patch.object(os, "chmod"), mock.patch.object(pathlib.Path, "chmod"): + with self.assertRaises(exception_type): + try: + rmtree(td) + except SkipTest as ex: + self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + # FIXME: Mark only the /proc-prefixing cases xfail, somehow (or fix them). @pytest.mark.xfail( reason="Many return paths prefixed /proc/cygdrive instead.", From 2a32e25bbd1cb42878928ef57a57100a17366202 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Oct 2023 02:30:22 -0400 Subject: [PATCH 0751/1790] Fix test bug that assumed staticmethod callability staticmethod objects are descriptors that cause the same-named attribute on a class or its instances to be callable (and the first argument, representing a class or instance, not to be passed). But the actual staticmethod objects themselves are only callable starting in Python 3.10. This reorgnizes the just-added test code so it no longer wrongly relies on being able to call such objects. --- test/test_util.py | 49 +++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index a852eb975..b108dc146 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -80,6 +80,30 @@ def __repr__(self): return "TestIterableMember(%r)" % self.name +@contextlib.contextmanager +def _tmpdir_to_force_permission_error(): + """Context manager to test permission errors in situations where we do not fix them.""" + if sys.platform == "cygwin": + raise SkipTest("Cygwin can't set the permissions that make the test meaningful.") + if sys.version_info < (3, 8): + raise SkipTest("In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") + + with tempfile.TemporaryDirectory() as parent: + td = pathlib.Path(parent, "testdir") + td.mkdir() + (td / "x").write_bytes(b"") + (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. + td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. + yield td + + +@contextlib.contextmanager +def _tmpdir_for_file_not_found(): + """Context manager to test errors deleting a directory that are not due to permissions.""" + with tempfile.TemporaryDirectory() as parent: + yield pathlib.Path(parent, "testdir") # It is deliberately never created. + + @ddt.ddt class TestUtils(TestBase): def setup(self): @@ -107,6 +131,7 @@ def test_rmtree_deletes_nested_dir_with_files(self): @skipIf(sys.platform == "cygwin", "Cygwin can't set the permissions that make the test meaningful.") def test_rmtree_deletes_dir_with_readonly_files(self): # Automatically works on Unix, but requires special handling on Windows. + # Not to be confused with _tmpdir_to_force_permission_error (which is used below). with tempfile.TemporaryDirectory() as parent: td = pathlib.Path(parent, "testdir") for d in td, td / "sub": @@ -122,31 +147,9 @@ def test_rmtree_deletes_dir_with_readonly_files(self): self.assertFalse(td.exists()) - @staticmethod - @contextlib.contextmanager - def _tmpdir_to_force_permission_error(): - if sys.platform == "cygwin": - raise SkipTest("Cygwin can't set the permissions that make the test meaningful.") - if sys.version_info < (3, 8): - raise SkipTest("In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") - - with tempfile.TemporaryDirectory() as parent: - td = pathlib.Path(parent, "testdir") - td.mkdir() - (td / "x").write_bytes(b"") - (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. - td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. - yield td - - @staticmethod - @contextlib.contextmanager - def _tmpdir_for_file_not_found(): - with tempfile.TemporaryDirectory() as parent: - yield pathlib.Path(parent, "testdir") # It is deliberately never created. - def test_rmtree_can_wrap_exceptions(self): """Our rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" - with self._tmpdir_to_force_permission_error() as td: + with _tmpdir_to_force_permission_error() as td: # Access the module through sys.modules so it is unambiguous which module's # attribute we patch: the original git.util, not git.index.util even though # git.index.util "replaces" git.util and is what "import git.util" gives us. From b8e009e8d31e32a2c0e247e0a7dc41ccdd3556e7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 13:59:03 -0400 Subject: [PATCH 0752/1790] In rmtree, have onerror catch only PermissionError The onerror function is still called on, and tries to resolve, any exception. But now, when it re-calls the file deletion function passed as func, the only exception it catches to conditionally convert to SkipTest is PermissionError (or derived exceptions). The old behavior of catching Exception was overly broad, and inconsistent with the hard-coded prefix of "FIXME: fails with: PermissionError" used to build the SkipTest exception messages. This commit also changes the message to use an f-string (which was one of the styles in the equivalent but differently coded duplicate logic eliminated in 5039df3, and seems clearer in this case). That change is a pure refactoring, not affecting generated messages. --- git/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/util.py b/git/util.py index 48901ba0c..c0e40d7e6 100644 --- a/git/util.py +++ b/git/util.py @@ -188,11 +188,11 @@ def onerror(func: Callable, path: PathLike, exc_info: str) -> None: try: func(path) # Will scream if still not possible to delete. - except Exception as ex: + except PermissionError as ex: if HIDE_WINDOWS_KNOWN_ERRORS: from unittest import SkipTest - raise SkipTest("FIXME: fails with: PermissionError\n {}".format(ex)) from ex + raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex raise return shutil.rmtree(path, False, onerror) From ccbb2732efcfa265568f1a535a8b746ed07ed82a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 18:14:19 -0400 Subject: [PATCH 0753/1790] Fix onerror callback type hinting, improve style The onerror parameter of shutil.rmtree receives a 3-tuple like what sys.exc_info() gives, not a string. Since we are not using that parameter anyway, I've fixed it in the onerror function defined in git.util.rmtree by changing it to Any rather than hinting it narrowly. I've also renamed the parameters so the names are based on those that are documented in the shutil.rmtree documentation. The names are not part of the function's interface (this follows both from the documentation and the typeshed hints) but using those names may make it easier to understand the function. - func is renamed to function. - path remains path. - exc_info is renamed to _excinfo. This parameter is documented as excinfo, but I've prepended an underscore to signifity that it is unused. These changes are to a local function and non-breaking. Finally, although not all type checkers will flag this as an error automatically, the git.util.rmtree function, like the shutil.rmtree function it calls, is conceptually void, so it should not have any return statements with operands. Because the return statement appeared at the end, I've removed "return". --- git/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/util.py b/git/util.py index c0e40d7e6..6e1e95e49 100644 --- a/git/util.py +++ b/git/util.py @@ -182,12 +182,12 @@ def rmtree(path: PathLike) -> None: :note: we use shutil rmtree but adjust its behaviour to see whether files that couldn't be deleted are read-only. Windows will not remove them in that case""" - def onerror(func: Callable, path: PathLike, exc_info: str) -> None: + def onerror(function: Callable, path: PathLike, _excinfo: Any) -> None: # Is the error an access error ? os.chmod(path, stat.S_IWUSR) try: - func(path) # Will scream if still not possible to delete. + function(path) # Will scream if still not possible to delete. except PermissionError as ex: if HIDE_WINDOWS_KNOWN_ERRORS: from unittest import SkipTest @@ -195,7 +195,7 @@ def onerror(func: Callable, path: PathLike, exc_info: str) -> None: raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex raise - return shutil.rmtree(path, False, onerror) + shutil.rmtree(path, False, onerror) def rmfile(path: PathLike) -> None: From 0b88012471d8021fffe61beb9d058840c0235f5d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 20:16:38 -0400 Subject: [PATCH 0754/1790] Use onexc callback where supported The shutil.rmtree callback defined as a local function in git.util.rmtree was already capable of being used as both the old onerror parameter and the new onexc parameter--introduced in Python 3.12, which also deprecates onerror--because they differ only in the meaning of their third parameter (excinfo), which the callback defined in git.util.rmtree does not use. This modifies git.util.rmtree to pass it as onexc on 3.12 and later, while still passing it as onerror on 3.11 and earlier. Because the default value of ignore_errors is False, this makes the varying logic clearer by omitting that argument and using a keyword argument both when passing onexc (which is keyword-only) and when passing onerror (which is not keyword-only but can only be passed positionally if ignore_errors is passed explicitly). --- git/util.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/git/util.py b/git/util.py index 6e1e95e49..8a4a20253 100644 --- a/git/util.py +++ b/git/util.py @@ -5,24 +5,24 @@ # the BSD License: https://opensource.org/license/bsd-3-clause/ from abc import abstractmethod -import os.path as osp -from .compat import is_win import contextlib from functools import wraps import getpass import logging import os +import os.path as osp +import pathlib import platform -import subprocess import re import shutil import stat -from sys import maxsize +import subprocess +import sys import time from urllib.parse import urlsplit, urlunsplit import warnings -# from git.objects.util import Traversable +from .compat import is_win # typing --------------------------------------------------------- @@ -42,22 +42,17 @@ Tuple, TypeVar, Union, - cast, TYPE_CHECKING, + cast, overload, ) -import pathlib - if TYPE_CHECKING: from git.remote import Remote from git.repo.base import Repo from git.config import GitConfigParser, SectionConstraint from git import Git - # from git.objects.base import IndexObject - - from .types import ( Literal, SupportsIndex, @@ -75,7 +70,6 @@ # --------------------------------------------------------------------- - from gitdb.util import ( # NOQA @IgnorePep8 make_sha, LockedFD, # @UnusedImport @@ -88,7 +82,6 @@ hex_to_bin, # @UnusedImport ) - # NOTE: Some of the unused imports might be used/imported by others. # Handle once test-cases are back up and running. # Most of these are unused here, but are for use by git-python modules so these @@ -182,7 +175,7 @@ def rmtree(path: PathLike) -> None: :note: we use shutil rmtree but adjust its behaviour to see whether files that couldn't be deleted are read-only. Windows will not remove them in that case""" - def onerror(function: Callable, path: PathLike, _excinfo: Any) -> None: + def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: # Is the error an access error ? os.chmod(path, stat.S_IWUSR) @@ -195,7 +188,10 @@ def onerror(function: Callable, path: PathLike, _excinfo: Any) -> None: raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex raise - shutil.rmtree(path, False, onerror) + if sys.version_info >= (3, 12): + shutil.rmtree(path, onexc=handler) + else: + shutil.rmtree(path, onerror=handler) def rmfile(path: PathLike) -> None: @@ -995,7 +991,7 @@ def __init__( self, file_path: PathLike, check_interval_s: float = 0.3, - max_block_time_s: int = maxsize, + max_block_time_s: int = sys.maxsize, ) -> None: """Configure the instance From 7dd59043b44d0f2169304f90da74b1b2f7f2b02e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Oct 2023 20:27:08 -0400 Subject: [PATCH 0755/1790] Revise and update rmtree docstrings and comments --- git/util.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/git/util.py b/git/util.py index 8a4a20253..d7f348d48 100644 --- a/git/util.py +++ b/git/util.py @@ -170,17 +170,18 @@ def patch_env(name: str, value: str) -> Generator[None, None, None]: def rmtree(path: PathLike) -> None: - """Remove the given recursively. + """Remove the given directory tree recursively. - :note: we use shutil rmtree but adjust its behaviour to see whether files that - couldn't be deleted are read-only. Windows will not remove them in that case""" + :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files that + couldn't be deleted are read-only. Windows will not remove them in that case.""" def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: - # Is the error an access error ? + """Callback for :func:`shutil.rmtree`. Works either as ``onexc`` or ``onerror``.""" + # Is the error an access error? os.chmod(path, stat.S_IWUSR) try: - function(path) # Will scream if still not possible to delete. + function(path) except PermissionError as ex: if HIDE_WINDOWS_KNOWN_ERRORS: from unittest import SkipTest From 196cfbeefb5ab0844d37da0b1e010b6ee7cf9041 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Oct 2023 03:27:58 -0400 Subject: [PATCH 0756/1790] Clean up test_util, reorganizing for readability - Slightly improve import sorting, grouping, and formatting. - Move the cygpath pairs parameters into the test class, so they can be immediately above the tests that use them. This was almost the case in the past, but stopped being the case when helpers for some new tests above those were introduced (and those helpers can't be moved inside the class without extra complexity). - Rename TestIterableMember to _Member, so it is no longer named as a test class. The unittest framework wouldn't consider it one, since it doesn't derive from unittest.TestCase, but the pytest runner, which we're actually using, does. More importanly (since it has no test methods anyway), this makes clear to humans that it is a helper class for tests, rather than a class of tests. - Improve the style of _Member, and have its __repr__ show the actual class of the instance, so if future tests ever use a derived class of it--or if its name ever changes again--the type name in the repr will be correct. - Remove the setup method (of TestUtils). It looks like this may at one time have been intended as a setUp method (note the case difference), but it is unused and there doesn't seem to be any attempt to use the instance attribute it was setting. - Use R"" instead of r"" for raw strings representing Windows paths, so that some editors (at least VS Code) refrain from highlighting their contents as regular expressions. - Other very minor reformatting and slight comment rewording. --- test/test_util.py | 101 +++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 55 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index b108dc146..7b37918d1 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -5,6 +5,7 @@ # the BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib +from datetime import datetime import os import pathlib import pickle @@ -13,7 +14,6 @@ import tempfile import time from unittest import SkipTest, mock, skipIf, skipUnless -from datetime import datetime import ddt import pytest @@ -28,10 +28,6 @@ utctz_to_altz, verify_utctz, ) -from test.lib import ( - TestBase, - with_rw_repo, -) from git.util import ( Actor, BlockingLockFile, @@ -43,41 +39,19 @@ remove_password_if_present, rmtree, ) +from test.lib import TestBase, with_rw_repo -_norm_cygpath_pairs = ( - (r"foo\bar", "foo/bar"), - (r"foo/bar", "foo/bar"), - (r"C:\Users", "/cygdrive/c/Users"), - (r"C:\d/e", "/cygdrive/c/d/e"), - ("C:\\", "/cygdrive/c/"), - (r"\\server\C$\Users", "//server/C$/Users"), - (r"\\server\C$", "//server/C$"), - ("\\\\server\\c$\\", "//server/c$/"), - (r"\\server\BAR/", "//server/BAR/"), - (r"D:/Apps", "/cygdrive/d/Apps"), - (r"D:/Apps\fOO", "/cygdrive/d/Apps/fOO"), - (r"D:\Apps/123", "/cygdrive/d/Apps/123"), -) - -_unc_cygpath_pairs = ( - (r"\\?\a:\com", "/cygdrive/a/com"), - (r"\\?\a:/com", "/cygdrive/a/com"), - (r"\\?\UNC\server\D$\Apps", "//server/D$/Apps"), -) - - -class TestIterableMember(object): - - """A member of an iterable list""" +class _Member: + """A member of an IterableList.""" - __slots__ = "name" + __slots__ = ("name",) def __init__(self, name): self.name = name def __repr__(self): - return "TestIterableMember(%r)" % self.name + return f"{type(self).__name__}({self.name!r})" @contextlib.contextmanager @@ -106,13 +80,6 @@ def _tmpdir_for_file_not_found(): @ddt.ddt class TestUtils(TestBase): - def setup(self): - self.testdict = { - "string": "42", - "int": 42, - "array": [42], - } - def test_rmtree_deletes_nested_dir_with_files(self): with tempfile.TemporaryDirectory() as parent: td = pathlib.Path(parent, "testdir") @@ -131,7 +98,7 @@ def test_rmtree_deletes_nested_dir_with_files(self): @skipIf(sys.platform == "cygwin", "Cygwin can't set the permissions that make the test meaningful.") def test_rmtree_deletes_dir_with_readonly_files(self): # Automatically works on Unix, but requires special handling on Windows. - # Not to be confused with _tmpdir_to_force_permission_error (which is used below). + # Not to be confused with what _tmpdir_to_force_permission_error sets up (see below). with tempfile.TemporaryDirectory() as parent: td = pathlib.Path(parent, "testdir") for d in td, td / "sub": @@ -179,6 +146,27 @@ def test_rmtree_does_not_wrap_unless_called_for(self, case): except SkipTest as ex: self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + _norm_cygpath_pairs = ( + (R"foo\bar", "foo/bar"), + (R"foo/bar", "foo/bar"), + (R"C:\Users", "/cygdrive/c/Users"), + (R"C:\d/e", "/cygdrive/c/d/e"), + ("C:\\", "/cygdrive/c/"), + (R"\\server\C$\Users", "//server/C$/Users"), + (R"\\server\C$", "//server/C$"), + ("\\\\server\\c$\\", "//server/c$/"), + (R"\\server\BAR/", "//server/BAR/"), + (R"D:/Apps", "/cygdrive/d/Apps"), + (R"D:/Apps\fOO", "/cygdrive/d/Apps/fOO"), + (R"D:\Apps/123", "/cygdrive/d/Apps/123"), + ) + + _unc_cygpath_pairs = ( + (R"\\?\a:\com", "/cygdrive/a/com"), + (R"\\?\a:/com", "/cygdrive/a/com"), + (R"\\?\UNC\server\D$\Apps", "//server/D$/Apps"), + ) + # FIXME: Mark only the /proc-prefixing cases xfail, somehow (or fix them). @pytest.mark.xfail( reason="Many return paths prefixed /proc/cygdrive instead.", @@ -192,16 +180,16 @@ def test_cygpath_ok(self, case): self.assertEqual(cwpath, cpath, wpath) @pytest.mark.xfail( - reason=r'2nd example r".\bar" -> "bar" fails, returns "./bar"', + reason=R'2nd example r".\bar" -> "bar" fails, returns "./bar"', raises=AssertionError, ) @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.data( - (r"./bar", "bar"), - (r".\bar", "bar"), # FIXME: Mark only this one xfail, somehow (or fix it). - (r"../bar", "../bar"), - (r"..\bar", "../bar"), - (r"../bar/.\foo/../chu", "../bar/chu"), + (R"./bar", "bar"), + (R".\bar", "bar"), # FIXME: Mark only this one xfail, somehow (or fix it). + (R"../bar", "../bar"), + (R"..\bar", "../bar"), + (R"../bar/.\foo/../chu", "../bar/chu"), ) def test_cygpath_norm_ok(self, case): wpath, cpath = case @@ -210,12 +198,12 @@ def test_cygpath_norm_ok(self, case): @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") @ddt.data( - r"C:", - r"C:Relative", - r"D:Apps\123", - r"D:Apps/123", - r"\\?\a:rel", - r"\\share\a:rel", + R"C:", + R"C:Relative", + R"D:Apps\123", + R"D:Apps/123", + R"\\?\a:rel", + R"\\share\a:rel", ) def test_cygpath_invalids(self, wpath): cwpath = cygpath(wpath) @@ -380,15 +368,18 @@ def test_actor_from_string(self): Actor("name last another", "some-very-long-email@example.com"), ) - @ddt.data(("name", ""), ("name", "prefix_")) + @ddt.data( + ("name", ""), + ("name", "prefix_"), + ) def test_iterable_list(self, case): name, prefix = case ilist = IterableList(name, prefix) name1 = "one" name2 = "two" - m1 = TestIterableMember(prefix + name1) - m2 = TestIterableMember(prefix + name2) + m1 = _Member(prefix + name1) + m2 = _Member(prefix + name2) ilist.extend((m1, m2)) From 100ab989fcba0b1d1bd89b5b4b41ea5014da3d82 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Oct 2023 15:31:25 -0400 Subject: [PATCH 0757/1790] Add initial test_env_vars_for_windows_tests The new test method just verifies the current behavior of the HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS environment variables. This is so there is a test to modify when changing that behavior. The purpose of these tests is *not* to claim that the behavior of either variable is something code that uses GitPython can (or has ever been able to) rely on. This also adds pytest-subtests as a dependency, so multiple failures from the subtests can be seen in the same test run. --- test-requirements.txt | 1 + test/test_util.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 9414da09c..a69181be1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,4 +7,5 @@ pre-commit pytest pytest-cov pytest-instafail +pytest-subtests pytest-sugar diff --git a/test/test_util.py b/test/test_util.py index 7b37918d1..d65b25d49 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -4,12 +4,14 @@ # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ +import ast import contextlib from datetime import datetime import os import pathlib import pickle import stat +import subprocess import sys import tempfile import time @@ -502,3 +504,46 @@ def test_remove_password_from_command_line(self): assert cmd_4 == remove_password_if_present(cmd_4) assert cmd_5 == remove_password_if_present(cmd_5) + + @ddt.data("HIDE_WINDOWS_KNOWN_ERRORS", "HIDE_WINDOWS_FREEZE_ERRORS") + def test_env_vars_for_windows_tests(self, name): + def run_parse(value): + command = [ + sys.executable, + "-c", + f"from git.util import {name}; print(repr({name}))", + ] + output = subprocess.check_output( + command, + env=None if value is None else dict(os.environ, **{name: value}), + text=True, + ) + return ast.literal_eval(output) + + assert_true_iff_win = self.assertTrue if os.name == "nt" else self.assertFalse + + truthy_cases = [ + ("unset", None), + ("true-seeming", "1"), + ("true-seeming", "true"), + ("true-seeming", "True"), + ("true-seeming", "yes"), + ("true-seeming", "YES"), + ("false-seeming", "0"), + ("false-seeming", "false"), + ("false-seeming", "False"), + ("false-seeming", "no"), + ("false-seeming", "NO"), + ("whitespace", " "), + ] + falsy_cases = [ + ("empty", ""), + ] + + for msg, env_var_value in truthy_cases: + with self.subTest(msg, env_var_value=env_var_value): + assert_true_iff_win(run_parse(env_var_value)) + + for msg, env_var_value in falsy_cases: + with self.subTest(msg, env_var_value=env_var_value): + self.assertFalse(run_parse(env_var_value)) From 7604da185ce12b9ef540aff3255580db02c88268 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Oct 2023 16:06:49 -0400 Subject: [PATCH 0758/1790] Warn if HIDE_WINDOWS_*_ERRORS set in environment This warns if the HIDE_WINDOWS_KNOWN_ERRORS or HIDE_WINDOWS_FREEZE_ERRORS environment variables are set. These behave unexpectedly, including (and especially) in their effect on the same-named git.util module attributes, and neither their effects nor those of those attributes are documented in a way that would have supported code outside the project relying on their specific semantics. The new warning message characterizes their status as deprecated. - This is now the case for HIDE_WINDOWS_KNOWN_ERRORS, and almost so for the same-named attribute, whose existence (though not its meaning) can technically be relied on due to inclusion in `__all__` (which this does *not* change). - But the HIDE_WINDOWS_FREEZE_ERRORS attribute was never guaranteed even to exist, so technically neither it nor the same-named environment variable are not *even* deprecated. The attribute's presence has never been reflected in the public interface of any GitPython component in any way. However, these attributes are still used by the tests. Furthermore, in the case of HIDE_WINDOWS_KNOWN_ERRORS, setting it is the only way to disable the behavior of converting errors from some file deletion operations into SkipTest exceptions on Windows. Since that behavior has not yet changed, but is unlikely to be desired outside of testing, no *attributes* are deprecated at this time, and no effort to warn from accessing or using attributes is attempted. --- git/util.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/git/util.py b/git/util.py index d7f348d48..5ffe93e70 100644 --- a/git/util.py +++ b/git/util.py @@ -109,14 +109,28 @@ log = logging.getLogger(__name__) -# types############################################################ + +def _read_env_flag(name: str, default: bool) -> Union[bool, str]: + try: + value = os.environ[name] + except KeyError: + return default + + log.warning( + "The %s environment variable is deprecated. Its effect has never been documented and changes without warning.", + name, + ) + + # FIXME: This should always return bool, as that is how it is used. + # FIXME: This should treat some values besides "" as expressing falsehood. + return value #: We need an easy way to see if Appveyor TCs start failing, #: so the errors marked with this var are considered "acknowledged" ones, awaiting remedy, #: till then, we wish to hide them. -HIDE_WINDOWS_KNOWN_ERRORS = is_win and os.environ.get("HIDE_WINDOWS_KNOWN_ERRORS", True) -HIDE_WINDOWS_FREEZE_ERRORS = is_win and os.environ.get("HIDE_WINDOWS_FREEZE_ERRORS", True) +HIDE_WINDOWS_KNOWN_ERRORS = is_win and _read_env_flag("HIDE_WINDOWS_KNOWN_ERRORS", True) +HIDE_WINDOWS_FREEZE_ERRORS = is_win and _read_env_flag("HIDE_WINDOWS_FREEZE_ERRORS", True) # { Utility Methods From eb51277e73ca274e3948a3f009168d210dd587ca Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Oct 2023 17:01:46 -0400 Subject: [PATCH 0759/1790] Make HIDE_* attributes always bool For now, this doesn't change how the correspondng environment variables are interpreted, in terms of truth and falsehood. But it does *convert* them to bool, so that the values of the HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS attributes are always bools. It also updates the tests accordingly, to validate this behavior. --- git/util.py | 5 ++--- test/test_util.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/git/util.py b/git/util.py index 5ffe93e70..a6153f1f3 100644 --- a/git/util.py +++ b/git/util.py @@ -110,7 +110,7 @@ log = logging.getLogger(__name__) -def _read_env_flag(name: str, default: bool) -> Union[bool, str]: +def _read_env_flag(name: str, default: bool) -> bool: try: value = os.environ[name] except KeyError: @@ -121,9 +121,8 @@ def _read_env_flag(name: str, default: bool) -> Union[bool, str]: name, ) - # FIXME: This should always return bool, as that is how it is used. # FIXME: This should treat some values besides "" as expressing falsehood. - return value + return bool(value) #: We need an easy way to see if Appveyor TCs start failing, diff --git a/test/test_util.py b/test/test_util.py index d65b25d49..5912ea4a0 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -520,7 +520,7 @@ def run_parse(value): ) return ast.literal_eval(output) - assert_true_iff_win = self.assertTrue if os.name == "nt" else self.assertFalse + true_iff_win = os.name == "nt" # Same as is_win, but don't depend on that here. truthy_cases = [ ("unset", None), @@ -542,8 +542,8 @@ def run_parse(value): for msg, env_var_value in truthy_cases: with self.subTest(msg, env_var_value=env_var_value): - assert_true_iff_win(run_parse(env_var_value)) + self.assertIs(run_parse(env_var_value), true_iff_win) for msg, env_var_value in falsy_cases: with self.subTest(msg, env_var_value=env_var_value): - self.assertFalse(run_parse(env_var_value)) + self.assertIs(run_parse(env_var_value), False) From 333896b4447c56093fa4ae402a3d22491928ce29 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Oct 2023 17:29:04 -0400 Subject: [PATCH 0760/1790] Treat false-seeming HIDE_* env var values as false This changes how HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS environment variables, if present, are interpreted, so that values that strongly seem intuitivley to represent falsehood now do. Before, only the empty string was treated as false. Now: - "0", "false", "no", and their case variants, as well as the empty string, are treated as false. - The presence of leading and trailing whitespace in the value now longer changes the truth value it represents. For example, all-whitespace (e.g., a space) is treated as false. - Values other than the above false values, and the recognized true values "1", "true", "yes", and their variants, are still treated as true, but issue a warning about how they are unrecognied. --- git/util.py | 10 ++++++++-- test/test_util.py | 8 ++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/git/util.py b/git/util.py index a6153f1f3..97f461a83 100644 --- a/git/util.py +++ b/git/util.py @@ -121,8 +121,14 @@ def _read_env_flag(name: str, default: bool) -> bool: name, ) - # FIXME: This should treat some values besides "" as expressing falsehood. - return bool(value) + adjusted_value = value.strip().lower() + + if adjusted_value in {"", "0", "false", "no"}: + return False + if adjusted_value in {"1", "true", "yes"}: + return True + log.warning("%s has unrecognized value %r, treating as %r.", name, value, default) + return default #: We need an easy way to see if Appveyor TCs start failing, diff --git a/test/test_util.py b/test/test_util.py index 5912ea4a0..647a02833 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -529,15 +529,15 @@ def run_parse(value): ("true-seeming", "True"), ("true-seeming", "yes"), ("true-seeming", "YES"), + ] + falsy_cases = [ + ("empty", ""), + ("whitespace", " "), ("false-seeming", "0"), ("false-seeming", "false"), ("false-seeming", "False"), ("false-seeming", "no"), ("false-seeming", "NO"), - ("whitespace", " "), - ] - falsy_cases = [ - ("empty", ""), ] for msg, env_var_value in truthy_cases: From c11b36660382c713709b36bbca1da8a1acb3a4ec Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Oct 2023 17:52:27 -0400 Subject: [PATCH 0761/1790] Simplify HIDE_* env var test; add missing cases Now that the expected truth values are intuitive, it is no longer necessary to group them by result and include messages that acknowldge the unintuitive cases. This reorders them so that pairs (like "yes" and "no") appear together, removes the messages that are no longer necessary, and reduces test code duplication. This also adds cases to test leading/trailing whitespace in otherwise nonempty strings, so it is clearer what the test is asserting overall, and so a bug where lstrip or rstrip (or equivalent with a regex) were used instead strip would be caught. --- test/test_util.py | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 647a02833..a4c06b80e 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -520,30 +520,22 @@ def run_parse(value): ) return ast.literal_eval(output) - true_iff_win = os.name == "nt" # Same as is_win, but don't depend on that here. - - truthy_cases = [ - ("unset", None), - ("true-seeming", "1"), - ("true-seeming", "true"), - ("true-seeming", "True"), - ("true-seeming", "yes"), - ("true-seeming", "YES"), - ] - falsy_cases = [ - ("empty", ""), - ("whitespace", " "), - ("false-seeming", "0"), - ("false-seeming", "false"), - ("false-seeming", "False"), - ("false-seeming", "no"), - ("false-seeming", "NO"), - ] - - for msg, env_var_value in truthy_cases: - with self.subTest(msg, env_var_value=env_var_value): - self.assertIs(run_parse(env_var_value), true_iff_win) - - for msg, env_var_value in falsy_cases: - with self.subTest(msg, env_var_value=env_var_value): - self.assertIs(run_parse(env_var_value), False) + for env_var_value, expected_truth_value in ( + (None, os.name == "nt"), # True on Windows when the environment variable is unset. + ("", False), + (" ", False), + ("0", False), + ("1", os.name == "nt"), + ("false", False), + ("true", os.name == "nt"), + ("False", False), + ("True", os.name == "nt"), + ("no", False), + ("yes", os.name == "nt"), + ("NO", False), + ("YES", os.name == "nt"), + (" no ", False), + (" yes ", os.name == "nt"), + ): + with self.subTest(env_var_value=env_var_value): + self.assertIs(run_parse(env_var_value), expected_truth_value) From f0e15e8935580a4e4dc4ed6490c9e1b229493e19 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Oct 2023 04:20:09 -0400 Subject: [PATCH 0762/1790] Further cleanup in test_util (on new tests) The main change here is to move the tests of how the HIDE_* environment variables are treated up near the rmtree tests, since they although the behaviors being tested are separate, they are conceptually related, and also not entirely independent in that they both involve the HIDE_WINDOWS_KNOWN_ERRORS attribute. Other changes are mostly to formatting. --- test/test_util.py | 94 +++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index a4c06b80e..b20657fb1 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -87,7 +87,14 @@ def test_rmtree_deletes_nested_dir_with_files(self): td = pathlib.Path(parent, "testdir") for d in td, td / "q", td / "s": d.mkdir() - for f in td / "p", td / "q" / "w", td / "q" / "x", td / "r", td / "s" / "y", td / "s" / "z": + for f in ( + td / "p", + td / "q" / "w", + td / "q" / "x", + td / "r", + td / "s" / "y", + td / "s" / "z", + ): f.write_bytes(b"") try: @@ -97,7 +104,10 @@ def test_rmtree_deletes_nested_dir_with_files(self): self.assertFalse(td.exists()) - @skipIf(sys.platform == "cygwin", "Cygwin can't set the permissions that make the test meaningful.") + @skipIf( + sys.platform == "cygwin", + "Cygwin can't set the permissions that make the test meaningful.", + ) def test_rmtree_deletes_dir_with_readonly_files(self): # Automatically works on Unix, but requires special handling on Windows. # Not to be confused with what _tmpdir_to_force_permission_error sets up (see below). @@ -117,7 +127,7 @@ def test_rmtree_deletes_dir_with_readonly_files(self): self.assertFalse(td.exists()) def test_rmtree_can_wrap_exceptions(self): - """Our rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + """rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" with _tmpdir_to_force_permission_error() as td: # Access the module through sys.modules so it is unambiguous which module's # attribute we patch: the original git.util, not git.index.util even though @@ -135,12 +145,16 @@ def test_rmtree_can_wrap_exceptions(self): (True, FileNotFoundError, _tmpdir_for_file_not_found), ) def test_rmtree_does_not_wrap_unless_called_for(self, case): - """Our rmtree doesn't wrap non-PermissionError, nor when HIDE_WINDOWS_KNOWN_ERRORS is false.""" + """rmtree doesn't wrap non-PermissionError, nor if HIDE_WINDOWS_KNOWN_ERRORS is false.""" hide_windows_known_errors, exception_type, tmpdir_context_factory = case with tmpdir_context_factory() as td: # See comments in test_rmtree_can_wrap_exceptions regarding the patching done here. - with mock.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors): + with mock.patch.object( + sys.modules["git.util"], + "HIDE_WINDOWS_KNOWN_ERRORS", + hide_windows_known_errors, + ): with mock.patch.object(os, "chmod"), mock.patch.object(pathlib.Path, "chmod"): with self.assertRaises(exception_type): try: @@ -148,6 +162,41 @@ def test_rmtree_does_not_wrap_unless_called_for(self, case): except SkipTest as ex: self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + @ddt.data("HIDE_WINDOWS_KNOWN_ERRORS", "HIDE_WINDOWS_FREEZE_ERRORS") + def test_env_vars_for_windows_tests(self, name): + def run_parse(value): + command = [ + sys.executable, + "-c", + f"from git.util import {name}; print(repr({name}))", + ] + output = subprocess.check_output( + command, + env=None if value is None else dict(os.environ, **{name: value}), + text=True, + ) + return ast.literal_eval(output) + + for env_var_value, expected_truth_value in ( + (None, os.name == "nt"), # True on Windows when the environment variable is unset. + ("", False), + (" ", False), + ("0", False), + ("1", os.name == "nt"), + ("false", False), + ("true", os.name == "nt"), + ("False", False), + ("True", os.name == "nt"), + ("no", False), + ("yes", os.name == "nt"), + ("NO", False), + ("YES", os.name == "nt"), + (" no ", False), + (" yes ", os.name == "nt"), + ): + with self.subTest(env_var_value=env_var_value): + self.assertIs(run_parse(env_var_value), expected_truth_value) + _norm_cygpath_pairs = ( (R"foo\bar", "foo/bar"), (R"foo/bar", "foo/bar"), @@ -504,38 +553,3 @@ def test_remove_password_from_command_line(self): assert cmd_4 == remove_password_if_present(cmd_4) assert cmd_5 == remove_password_if_present(cmd_5) - - @ddt.data("HIDE_WINDOWS_KNOWN_ERRORS", "HIDE_WINDOWS_FREEZE_ERRORS") - def test_env_vars_for_windows_tests(self, name): - def run_parse(value): - command = [ - sys.executable, - "-c", - f"from git.util import {name}; print(repr({name}))", - ] - output = subprocess.check_output( - command, - env=None if value is None else dict(os.environ, **{name: value}), - text=True, - ) - return ast.literal_eval(output) - - for env_var_value, expected_truth_value in ( - (None, os.name == "nt"), # True on Windows when the environment variable is unset. - ("", False), - (" ", False), - ("0", False), - ("1", os.name == "nt"), - ("false", False), - ("true", os.name == "nt"), - ("False", False), - ("True", os.name == "nt"), - ("no", False), - ("yes", os.name == "nt"), - ("NO", False), - ("YES", os.name == "nt"), - (" no ", False), - (" yes ", os.name == "nt"), - ): - with self.subTest(env_var_value=env_var_value): - self.assertIs(run_parse(env_var_value), expected_truth_value) From a9b05ece674578d3417f8969ade17b06ab287ffe Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Oct 2023 07:59:32 -0400 Subject: [PATCH 0763/1790] Clarify a test helper docstring The wording was ambiguous before, because fixing a PermissionError could mean addressing it successfully at runtime (which is the intended meaning in that docstring) but it could also mean fixing a bug or test case related to such an error (not intended). --- test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index b20657fb1..f75231c98 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -58,7 +58,7 @@ def __repr__(self): @contextlib.contextmanager def _tmpdir_to_force_permission_error(): - """Context manager to test permission errors in situations where we do not fix them.""" + """Context manager to test permission errors in situations where they are not overcome.""" if sys.platform == "cygwin": raise SkipTest("Cygwin can't set the permissions that make the test meaningful.") if sys.version_info < (3, 8): From ebd9ce50ebd702b0655384db0d509113c70ad94c Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Tue, 10 Oct 2023 15:03:43 -0600 Subject: [PATCH 0764/1790] Fix unused import linting --- git/exc.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/git/exc.py b/git/exc.py index 936381fd8..66cf9ad06 100644 --- a/git/exc.py +++ b/git/exc.py @@ -5,16 +5,16 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php """ Module containing all exceptions thrown throughout the git package, """ -from gitdb.exc import ( - AmbiguousObjectName, # @UnusedImport - BadName, # @UnusedImport - BadObject, # @UnusedImport - BadObjectType, # @UnusedImport - InvalidDBRoot, # @UnusedImport - ODBError, # @UnusedImport - ParseError, # @UnusedImport - UnsupportedOperation, # @UnusedImport - to_hex_sha, # @UnusedImport +from gitdb.exc import ( # noqa: @UnusedImport + AmbiguousObjectName, + BadName, + BadObject, + BadObjectType, + InvalidDBRoot, + ODBError, + ParseError, + UnsupportedOperation, + to_hex_sha, ) from git.compat import safe_decode from git.util import remove_password_if_present From b1478bb1091e24dc001798da6d28bb366baf733f Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Tue, 10 Oct 2023 15:19:01 -0600 Subject: [PATCH 0765/1790] Typo --- git/exc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/exc.py b/git/exc.py index 66cf9ad06..5110d502d 100644 --- a/git/exc.py +++ b/git/exc.py @@ -3,7 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -""" Module containing all exceptions thrown throughout the git package, """ +""" Module containing all exceptions thrown throughout the git package """ from gitdb.exc import ( # noqa: @UnusedImport AmbiguousObjectName, From a4eb2b3ea7680ac4da01d72746d9e82aa0cb0422 Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Tue, 10 Oct 2023 15:24:15 -0600 Subject: [PATCH 0766/1790] black lint --- git/__init__.py | 150 ++++++++++++++++++++++++------------------------ 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 00e7b8dd3..ef7493079 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -64,81 +64,81 @@ def _init_externals() -> None: # __all__ must be statically defined by py.typed support # __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] __all__ = [ - 'Actor', - 'AmbiguousObjectName', - 'BadName', - 'BadObject', - 'BadObjectType', - 'BaseIndexEntry', - 'Blob', - 'BlobFilter', - 'BlockingLockFile', - 'CacheError', - 'CheckoutError', - 'CommandError', - 'Commit', - 'Diff', - 'DiffIndex', - 'Diffable', - 'FetchInfo', - 'Git', - 'GitCmdObjectDB', - 'GitCommandError', - 'GitCommandNotFound', - 'GitConfigParser', - 'GitDB', - 'GitError', - 'HEAD', - 'Head', - 'HookExecutionError', - 'IndexEntry', - 'IndexFile', - 'IndexObject', - 'InvalidDBRoot', - 'InvalidGitRepositoryError', - 'List', - 'LockFile', - 'NULL_TREE', - 'NoSuchPathError', - 'ODBError', - 'Object', - 'Optional', - 'ParseError', - 'PathLike', - 'PushInfo', - 'RefLog', - 'RefLogEntry', - 'Reference', - 'Remote', - 'RemoteProgress', - 'RemoteReference', - 'Repo', - 'RepositoryDirtyError', - 'RootModule', - 'RootUpdateProgress', - 'Sequence', - 'StageType', - 'Stats', - 'Submodule', - 'SymbolicReference', - 'TYPE_CHECKING', - 'Tag', - 'TagObject', - 'TagReference', - 'Tree', - 'TreeModifier', - 'Tuple', - 'Union', - 'UnmergedEntriesError', - 'UnsafeOptionError', - 'UnsafeProtocolError', - 'UnsupportedOperation', - 'UpdateProgress', - 'WorkTreeRepositoryUnsupported', - 'remove_password_if_present', - 'rmtree', - 'safe_decode', - 'to_hex_sha', + "Actor", + "AmbiguousObjectName", + "BadName", + "BadObject", + "BadObjectType", + "BaseIndexEntry", + "Blob", + "BlobFilter", + "BlockingLockFile", + "CacheError", + "CheckoutError", + "CommandError", + "Commit", + "Diff", + "DiffIndex", + "Diffable", + "FetchInfo", + "Git", + "GitCmdObjectDB", + "GitCommandError", + "GitCommandNotFound", + "GitConfigParser", + "GitDB", + "GitError", + "HEAD", + "Head", + "HookExecutionError", + "IndexEntry", + "IndexFile", + "IndexObject", + "InvalidDBRoot", + "InvalidGitRepositoryError", + "List", + "LockFile", + "NULL_TREE", + "NoSuchPathError", + "ODBError", + "Object", + "Optional", + "ParseError", + "PathLike", + "PushInfo", + "RefLog", + "RefLogEntry", + "Reference", + "Remote", + "RemoteProgress", + "RemoteReference", + "Repo", + "RepositoryDirtyError", + "RootModule", + "RootUpdateProgress", + "Sequence", + "StageType", + "Stats", + "Submodule", + "SymbolicReference", + "TYPE_CHECKING", + "Tag", + "TagObject", + "TagReference", + "Tree", + "TreeModifier", + "Tuple", + "Union", + "UnmergedEntriesError", + "UnsafeOptionError", + "UnsafeProtocolError", + "UnsupportedOperation", + "UpdateProgress", + "WorkTreeRepositoryUnsupported", + "remove_password_if_present", + "rmtree", + "safe_decode", + "to_hex_sha", ] # { Initialize git executable path From cc848d256369b01ffa9c6e1cb1c0001e6d828dcf Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 11 Oct 2023 04:27:52 -0400 Subject: [PATCH 0767/1790] Fix small #1662 regression due to #1659 When #1659 was updated to pick up linting configuration changes, it inadvertently undid one of the URL changes made in #1662, putting the URL in the git.exc module back to the one that redirects to a different BSD license from the one this project uses. Since only that one module was affected, the fix is simple. This only changes the URL back; it doesn't undo any other #1659 changes. --- git/exc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/exc.py b/git/exc.py index 5110d502d..b4ffe7568 100644 --- a/git/exc.py +++ b/git/exc.py @@ -2,7 +2,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php +# the BSD License: https://opensource.org/license/bsd-3-clause/ """ Module containing all exceptions thrown throughout the git package """ from gitdb.exc import ( # noqa: @UnusedImport From 7b1c04681a7dfcd2461be0b57f0914d15d81e782 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 01:45:06 -0400 Subject: [PATCH 0768/1790] Have Dependabot offer submodule updates This extends the current Dependabot configuration so that, in addition to offering updates for GitHub Actions, it also offers them for git submodules (for the gitdb direct submodule). --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8c139c7be..5acde1a9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,8 @@ updates: directory: "/" schedule: interval: "weekly" + +- package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "monthly" From a29a8750c914ac1d13fc784994132a077d47e686 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 02:23:07 -0400 Subject: [PATCH 0769/1790] Drop obsolete info on yanking from security policy Versions may still be yanked for security reasons under specific circumstances, but this is not the usual or most common practice in GitPython, at least currently. Recent security updates have not been accompanied by yanking older versions, and allowing these versions to be selected automatically even when not called for specifically can be good, such as to prevent an even older version with even more vulnerabilities from being selected in situations where for some reason the latest version cannot yet be used. In general, users shouldn't (and don't) assume all non-yanked versions to be free of security fixes that later versions have received. This change updates SECURITY.md to avoid giving that impression, but of course some versions of GitPython may still be yanked in the future if circumstances warrant it. --- SECURITY.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index cf25c09ea..cbfaafdde 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,8 +2,7 @@ ## Supported Versions -Only the latest version of GitPython can receive security updates. If a vulnerability is discovered, a fix can be issued in a new release, while older releases -are likely to be yanked. +Only the latest version of GitPython can receive security updates. If a vulnerability is discovered, a fix can be issued in a new release. | Version | Supported | | ------- | ------------------ | From 05e4ca31d33733ddb3b34272bbd04b6b9cc784f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 07:51:18 +0000 Subject: [PATCH 0770/1790] Bump git/ext/gitdb from `49c3178` to `8ec2390` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `49c3178` to `8ec2390`. - [Commits](https://github.com/gitpython-developers/gitdb/compare/49c3178711ddb3510f0e96297187f823cc019871...8ec23904f44fbbb9bb560b83bd891a9b48be7645) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 49c317871..8ec23904f 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 49c3178711ddb3510f0e96297187f823cc019871 +Subproject commit 8ec23904f44fbbb9bb560b83bd891a9b48be7645 From 12db86cefa90e8fc1c8df8f5c6b4c8b7a232d773 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 04:06:47 -0400 Subject: [PATCH 0771/1790] Have Dependabot update smmap submodule dependency This makes Dependabot open version update PRs for submodules (which here is just smmap), as well as GitHub Actions. This is like https://github.com/gitpython-developers/GitPython/pull/1702. --- .github/dependabot.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 203f3c889..5acde1a9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,9 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "weekly" + +- package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "monthly" From 8e613962153baad0431358dda03d9fd942cef616 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 08:17:45 +0000 Subject: [PATCH 0772/1790] Bump gitdb/ext/smmap from `334ef84` to `f1ace75` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `334ef84` to `f1ace75`. - [Commits](https://github.com/gitpython-developers/smmap/compare/334ef84a05c953ed5dbec7b9c6d4310879eeab5a...f1ace75be355fdec927793e462b9b12bf6ec9520) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 334ef84a0..f1ace75be 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 334ef84a05c953ed5dbec7b9c6d4310879eeab5a +Subproject commit f1ace75be355fdec927793e462b9b12bf6ec9520 From 24a45f21e0a93a4bf94b45e46d8b91b03632019a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 08:27:51 +0000 Subject: [PATCH 0773/1790] Bump git/ext/gitdb from `8ec2390` to `6a22706` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `8ec2390` to `6a22706`. - [Commits](https://github.com/gitpython-developers/gitdb/compare/8ec23904f44fbbb9bb560b83bd891a9b48be7645...6a2270635842c4287ae14d94fdc3f9e1ddd0527a) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 8ec23904f..6a2270635 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 8ec23904f44fbbb9bb560b83bd891a9b48be7645 +Subproject commit 6a2270635842c4287ae14d94fdc3f9e1ddd0527a From ba750deec95df77b7f0819e18fa02acbb52cecdf Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 17:15:16 -0400 Subject: [PATCH 0774/1790] Update readme for milestone-less releasing --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index b9e61912a..b567d49a2 100644 --- a/README.md +++ b/README.md @@ -199,13 +199,12 @@ Please have a look at the [contributions file][contributing]. ### How to make a new release - Update/verify the **version** in the `VERSION` file. -- Update/verify that the `doc/source/changes.rst` changelog file was updated. +- Update/verify that the `doc/source/changes.rst` changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` - Commit everything. - Run `git tag -s ` to tag the version in Git. - _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) using `venv` or `virtualenv`.\ (When run in a virtual environment, the next step will automatically take care of installing `build` and `twine` in it.) - Run `make release`. -- Close the milestone mentioned in the _changelog_ and create a new one. _Do not reuse milestones by renaming them_. - Go to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. ### How to verify a release (DEPRECATED) From a151ea0f66bee8f810b2f93c883a2a0dbad67616 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 17:53:29 -0400 Subject: [PATCH 0775/1790] Make the release instructions a numbered list Since they are sequential steps. --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b567d49a2..25d2a910e 100644 --- a/README.md +++ b/README.md @@ -198,14 +198,14 @@ Please have a look at the [contributions file][contributing]. ### How to make a new release -- Update/verify the **version** in the `VERSION` file. -- Update/verify that the `doc/source/changes.rst` changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` -- Commit everything. -- Run `git tag -s ` to tag the version in Git. -- _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) using `venv` or `virtualenv`.\ +1. Update/verify the **version** in the `VERSION` file. +2. Update/verify that the `doc/source/changes.rst` changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` +3. Commit everything. +4. Run `git tag -s ` to tag the version in Git. +5. _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) using `venv` or `virtualenv`.\ (When run in a virtual environment, the next step will automatically take care of installing `build` and `twine` in it.) -- Run `make release`. -- Go to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. +6. Run `make release`. +7. Go to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. ### How to verify a release (DEPRECATED) From feb4414d955cc76df496f406ae914793028a804e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 18:01:50 -0400 Subject: [PATCH 0776/1790] Shorten another step --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 25d2a910e..a7942fd2f 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,7 @@ Please have a look at the [contributions file][contributing]. 2. Update/verify that the `doc/source/changes.rst` changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` 3. Commit everything. 4. Run `git tag -s ` to tag the version in Git. -5. _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) using `venv` or `virtualenv`.\ -(When run in a virtual environment, the next step will automatically take care of installing `build` and `twine` in it.) +5. _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment). (Then the next step can install `build` and `twine`.) 6. Run `make release`. 7. Go to [GitHub Releases](https://github.com/gitpython-developers/GitPython/releases) and publish a new one with the recently pushed tag. Generate the changelog. From e8956e53e75d59d1bda1c69307e19ae79921d813 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 19:48:44 -0400 Subject: [PATCH 0777/1790] Run Cygwin CI workflow commands in login shells This passes --login to the bash shell used to run commands in the Cygwin environment on CI. This eliminates the need to work around a partly broken environment, and the extra code what was used to do that is accordingly removed. There are two benefits of this change: - The PATH is correct: Cygwin's /usr/local/bin and /usr/bin are present at the beginning of PATH. Otherwise, it is easy to get /usr/bin at the front, but rather involved to get /usr/local/bin to precede it. Because Python on Cygwin puts scripts/executables such as the upgraded "pip" and the "pytest" command in /usr/local/bin, it is valuable to have that directory in the PATH and best to have it before /usr/bin. (I have set CYGWIN_NOWINPATH to omit other directories, since finding any of the commands to be run in the Cygwin environment outside that environment is unintended.) - Every step automatically has correct temporary directories: When Cygwin commands were not being run in login shells, they didn't automatically get correct values for TMP and TEMP for their environment. To work around this, those environment variables were set globally, for every step. But that caused them to refer to nonexistent locations for steps such as actions/checkout. Most likely this would not cause any errors, but it did cause copious warnings about a nonexistent temporary directory, which risked obscuring other potentially important output. Now that Cygwin commands run in login shells, both the few non-Cygwin steps, and the steps run in the Cygwin enviroment, all get correct temporary directories (with TMP and TEMP set in the prewritten startup script the login shell uses). A theoretical disadvantage of this is that login shells take slightly longer to start up, but that delay is insigificant in this application. A more significant disadvantage is that setting the -x shell option the way it was done before would produce a lot of noise at the beginning of the output for every command-running step. To work around that, -x is omitted from the value of "shell" and "set -x" is added at the end of the startup script for login shells, so it runs before each step's "payload" command, but without applying to the commands run in the startup script itself. --- .github/workflows/cygwin-test.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index cd913385f..d190f1132 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -10,30 +10,36 @@ jobs: fail-fast: false env: - CHERE_INVOKING: 1 - TMP: "/tmp" - TEMP: "/tmp" + CHERE_INVOKING: "1" + CYGWIN_NOWINPATH: "1" defaults: run: - shell: C:\cygwin\bin\bash.exe --noprofile --norc -exo pipefail -o igncr "{0}" + shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" steps: - name: Force LF line endings run: | git config --global core.autocrlf false # Affects the non-Cygwin git. - shell: bash + shell: bash # Use Git Bash instead of Cygwin Bash for this step. - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: recursive - - uses: cygwin/cygwin-install-action@v4 + - name: Install Cygwin + uses: cygwin/cygwin-install-action@v4 with: packages: python39 python39-pip python39-virtualenv git + add-to-path: false # No need to change $PATH outside the Cygwin environment. - - name: Special configuration for Cygwin's git + - name: Arrange for verbose output + run: | + # Arrange for verbose output but without shell environment setup details. + echo 'set -x' >~/.bash_profile + + - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" git config --global core.autocrlf false @@ -57,7 +63,7 @@ jobs: - name: Install project and test dependencies run: | - python -m pip install ".[test]" + pip install ".[test]" - name: Show version and platform information run: | @@ -71,4 +77,4 @@ jobs: - name: Test with pytest run: | - python -m pytest --color=yes -p no:sugar --instafail -vv + pytest --color=yes -p no:sugar --instafail -vv From 6f765a2028e45a14853bf06801104cc6d22e053e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 17 Oct 2023 08:08:14 +0200 Subject: [PATCH 0778/1790] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ git/ext/gitdb | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 1f1a39706..3fef00337 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.37 +3.1.38 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index a789b068d..431ad03e2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.38 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.38 + 3.1.37 ====== diff --git a/git/ext/gitdb b/git/ext/gitdb index 6a2270635..8ec23904f 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 6a2270635842c4287ae14d94fdc3f9e1ddd0527a +Subproject commit 8ec23904f44fbbb9bb560b83bd891a9b48be7645 From 427c177047b526b5d15859e36b2c86fea477bfff Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 04:51:01 -0400 Subject: [PATCH 0779/1790] Add missing info in Submodule.remove docstring This rewords and adds some missing information to the docstring of Submodule.remove, for the "method" parameter, discussed in #1712. It uses the second suggestion presented in that issue, formatted in the style of the surrounding docstring (an 88-column wrap). It also does some other rewording in that docstring, for clarity, at the end of that same section (on the "method" parameter), and a small punctuation fix in the section about the "force" parameter. --- git/objects/submodule/base.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 13d897263..473ebde76 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -948,17 +948,16 @@ def remove( """Remove this submodule from the repository. This will remove our entry from the .gitmodules file and the entry in the .git / config file. - :param module: If True, the module checkout we point to will be deleted - as well. If the module is currently on a commit which is not part - of any branch in the remote, if the currently checked out branch - working tree, or untracked files, - is ahead of its tracking branch, if you have modifications in the + :param module: If True, the checked out module we point to will be deleted as + well.If that module is currently on a commit outside any branch in the + remote, or if it is ahead of its tracking branch, or if there are modified + or untracked files in its working tree, then the removal will fail. In case the removal of the repository fails for these reasons, the submodule status will not have been altered. - If this submodule has child - modules on its own, these will be deleted - prior to touching the own module. + If this submodule has child modules of its own, these will be deleted prior + to touching the direct submodule. :param force: Enforces the deletion of the module even though it contains - modifications. This basically enforces a brute - force file system based + modifications. This basically enforces a brute-force file system based deletion. :param configuration: if True, the submodule is deleted from the configuration, otherwise it isn't. Although this should be enabled most of the times, From bb48c8775a5415f2c86663e356067b53f414a0cc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 05:59:57 -0400 Subject: [PATCH 0780/1790] Don't pre-clone submodules on CI, at least for now This has actions/checkout no longer automatically clone submodules in the CI test workflows. This change is for the purpose of reproducing #1713, to allow the forthcoming fix for it to be tested. However, continuing to rely on init-tests-after-clone.sh to get the submodules would serve as a kind of regression testing for #1713. So it is unclear at this time if and when this change should be undone. --- .github/workflows/cygwin-test.yml | 1 - .github/workflows/pythonpackage.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index d190f1132..89c03a394 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -26,7 +26,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive - name: Install Cygwin uses: cygwin/cygwin-install-action@v4 diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 2a82e0e03..2dd97183b 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -27,7 +27,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 8ea3133bb15a4498cfa7ab256a0d79287ac564c9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 06:40:40 -0400 Subject: [PATCH 0781/1790] Have init script clone submodules unconditionally Since 7110bf8 (in #1693), "git submodule update --init --recursive" was not run on CI, on the mistaken grounds that the CI test workflows would already have taken care of cloning all submodules (ever since 4eef3ec when the "submodules: recursive" option was added to the actions/checkout step). This changes the init-tests-after-clone.sh script to again run that command unconditionally, including on CI. The assumption that it wasn't needed on CI was based on the specific content of GitPython's own GitHub Actions workflows. But this disregarded that the test suite is run on CI for *other* projects: specifically, for downstream projects that package GitPython (#1713). This also brings back the comment from fc96980 that says more about how the tests rely on submodules being present (specifically, that they need a submodule with a submodule). However, that is not specifically related to the bug being fixed. --- init-tests-after-clone.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 21d1f86d8..118e1de22 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -47,10 +47,8 @@ git reset --hard HEAD~1 # Point the master branch where we started, so we test the correct code. git reset --hard __testing_point__ -# The tests need submodules. (On CI, they would already have been checked out.) -if ! ci; then - git submodule update --init --recursive -fi +# The tests need submodules, including a submodule with a submodule. +git submodule update --init --recursive # The tests need some version tags. Try to get them even in forks. This fetches # other objects too. So, locally, we always do it, for a consistent experience. From 44102f30eaadcd122899f5f801f28b83bd9a5111 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Oct 2023 17:26:05 +0200 Subject: [PATCH 0782/1790] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3fef00337..efb1eb44f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.38 +3.1.40 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 431ad03e2..71f0bd74c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.40 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.40 + 3.1.38 ====== From 8197e9043f26a88168dc3227c4fbb0ec64ef42f9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 19 Oct 2023 21:17:19 -0400 Subject: [PATCH 0783/1790] Remove now-unused import in top-level __init__.py The inspect module was only used to dynamically generate __all__, which is statically written since c862845. Although it includes everything it originally had for compatibility (8edc53b), this did not include modules, since the comprehension used to generate it omitted those (which is what it needed the inspect module for). So it was not, and is not, listed in __all__, and can be removed. --- git/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/git/__init__.py b/git/__init__.py index be8338ddc..38a59e517 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -6,7 +6,6 @@ # flake8: noqa # @PydevCodeAnalysisIgnore from git.exc import * # @NoMove @IgnorePep8 -import inspect import os import sys import os.path as osp From 7545b802d811128ac367a47b794917f2f678836b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 19 Oct 2023 21:41:39 -0400 Subject: [PATCH 0784/1790] Add __all__ in git.exc, adjust __init__.py imports The git.exc module imports exceptions from gitdb.exc to republish them, as well as defining its own (also for use from outside). But because it did not define __all__, the intent for the exceptions it imported was unclear, since names that are introduced by imports and not present in __all__ are not generally considered public, even when __all__ is absent and a "*" import would reimport them. This rectifies that by adding __all__ and listing both imported and newly introduced exceptions explicitly in it. Although this strictly expands which names are public under typical conventions, it strictly contracts which names are imported by a "*" import, because the presence of __all__ suppresses names not listed in it from being imported that way. However, because under typical conventions those other names are not considered public, and they were not even weakly documented as public, this should be okay. (Even though this is not a breaking change, in that code it would break would already technically be broken... if it turns out that it is common to wrongly rely on the availabiliy of those names, then this may need to be revisited and slightly modified.) This brings the readily identified public interface of git.exc in line with what is weakly implied (and intended) by its docstring. This also modifies __init__.py accordingly: The top-level git module has for some time used a "*" import on git.exc, causing the extra names originally meant as implementation details to be included. Because its own __all__ was dynamically generated until c862845, #1659 also added 8edc53b to retain the formerly present names in __all__. So the change here imports those names from the modules that deliberately provide them, to preserve compatibility. --- git/__init__.py | 8 ++++++-- git/exc.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 38a59e517..05a02a7ff 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -7,10 +7,10 @@ # @PydevCodeAnalysisIgnore from git.exc import * # @NoMove @IgnorePep8 import os -import sys import os.path as osp +import sys -from typing import Optional +from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING from git.types import PathLike __version__ = "git" @@ -38,7 +38,10 @@ def _init_externals() -> None: # { Imports +from gitdb.util import to_hex_sha + try: + from git.compat import safe_decode # @NoMove @IgnorePep8 from git.config import GitConfigParser # @NoMove @IgnorePep8 from git.objects import * # @NoMove @IgnorePep8 from git.refs import * # @NoMove @IgnorePep8 @@ -53,6 +56,7 @@ def _init_externals() -> None: BlockingLockFile, Stats, Actor, + remove_password_if_present, rmtree, ) except GitError as _exc: diff --git a/git/exc.py b/git/exc.py index b4ffe7568..32c371d0b 100644 --- a/git/exc.py +++ b/git/exc.py @@ -5,7 +5,34 @@ # the BSD License: https://opensource.org/license/bsd-3-clause/ """ Module containing all exceptions thrown throughout the git package """ -from gitdb.exc import ( # noqa: @UnusedImport +__all__ = [ + # Defined in gitdb.exc: + "AmbiguousObjectName", + "BadName", + "BadObject", + "BadObjectType", + "InvalidDBRoot", + "ODBError", + "ParseError", + "UnsupportedOperation", + # Introduced in this module: + "GitError", + "InvalidGitRepositoryError", + "WorkTreeRepositoryUnsupported", + "NoSuchPathError", + "UnsafeProtocolError", + "UnsafeOptionError", + "CommandError", + "GitCommandNotFound", + "GitCommandError", + "CheckoutError", + "CacheError", + "UnmergedEntriesError", + "HookExecutionError", + "RepositoryDirtyError", +] + +from gitdb.exc import ( AmbiguousObjectName, BadName, BadObject, @@ -14,7 +41,6 @@ ODBError, ParseError, UnsupportedOperation, - to_hex_sha, ) from git.compat import safe_decode from git.util import remove_password_if_present From 2af36792db870c6336b52d3c8aa3d19c0b2fe9c4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 02:56:16 -0400 Subject: [PATCH 0785/1790] Remove `@UnusedImport` from an import that is used Inclusion in __all__ is considered a use. --- git/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/db.py b/git/db.py index bff43347b..b1a0d108a 100644 --- a/git/db.py +++ b/git/db.py @@ -1,7 +1,7 @@ """Module with our own gitdb implementation - it uses the git command""" from git.util import bin_to_hex, hex_to_bin from gitdb.base import OInfo, OStream -from gitdb.db import GitDB # @UnusedImport +from gitdb.db import GitDB from gitdb.db import LooseObjectDB from gitdb.exc import BadObject From 2057ae67b85fb9925efbd0f00f44413e506e286c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 20 Oct 2023 09:37:58 +0200 Subject: [PATCH 0786/1790] bump versions to prepare for next release --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 7d85ea55b..0b8de13d5 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +4.0.11 +****** + +- various improvements - please see the release on GitHub for details. + ****** 4.0.10 ****** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2fb3f7edb..803a4283f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -28,7 +28,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 10) +version_info = (4, 0, 11) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f1ace75be..256c5a21d 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f1ace75be355fdec927793e462b9b12bf6ec9520 +Subproject commit 256c5a21de2d14aca02c9689d7d63f78c4e0ef61 diff --git a/setup.py b/setup.py index 61b572768..f67f7a58f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 10) +version_info = (4, 0, 11) __version__ = '.'.join(str(i) for i in version_info) setup( From 3d3e9572dc452fea53d328c101b3d1440bbefe40 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 20 Oct 2023 09:42:19 +0200 Subject: [PATCH 0787/1790] fix makefile to allow building packages with current python version It all breaks all the time, it's just like that. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7aa5a717a..a0a2d0e01 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ release:: clean force_release:: clean git push --tags - python3 setup.py sdist bdist_wheel + python3 -m build --sdist --wheel twine upload dist/* doc:: From 965d2d36703a60f610e4b4a3fb1b86fe244d63d5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 06:46:10 -0400 Subject: [PATCH 0788/1790] Never add a vendored smmap directory to sys.path This removes the logic that appended the git submodule directory for smmap to sys.path under most circumstances when the version of gitdb was not from PyPI. Now gitdb does not modify sys.path. See https://github.com/gitpython-developers/GitPython/issues/1717 and https://github.com/gitpython-developers/GitPython/pull/1720 for context. This change is roughly equivalent to the change to GitPython, though as noted the behavior being eliminated is subtly different here and there. --- gitdb/__init__.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 803a4283f..9b77e9f2d 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -4,34 +4,12 @@ # the New BSD License: https://opensource.org/license/bsd-3-clause/ """Initialize the object database module""" -import sys -import os - -#{ Initialization - - -def _init_externals(): - """Initialize external projects by putting them into the path""" - if 'PYOXIDIZER' not in os.environ: - where = os.path.join(os.path.dirname(__file__), 'ext', 'smmap') - if os.path.exists(where): - sys.path.append(where) - - import smmap - del smmap - # END handle imports - -#} END initialization - -_init_externals() - __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" version_info = (4, 0, 11) __version__ = '.'.join(str(i) for i in version_info) - # default imports from gitdb.base import * from gitdb.db import * From dfbfb12beee6b2c61cb02f193fabc427e0a949f6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 07:24:33 -0400 Subject: [PATCH 0789/1790] Revise and update the readme Changes worth mentioning: - Format commands as code blocks instead of blockquotes. (This is particularly useful for the submodule update step, whose lines were inadvertently concatenated, but it also improves appearance overall.) - Mention smmap as a requirement. (But also that it doesn't need to be separately installed.) - Mention that gitdb-speedups is not currently maintained. - No longer say gitdb has source code in gitorious. (Since that site no longer exists.) - Call GitPython "GitPython" rather than "git-python". - Replace the old git-python Google Groups link with a link to the Discussions page on the GitHub repository for GitPython. (This seems like the closest currently available resource.) --- README.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 29c70f781..61ce28b1e 100644 --- a/README.rst +++ b/README.rst @@ -16,34 +16,38 @@ Installation :target: https://readthedocs.org/projects/gitdb/?badge=latest :alt: Documentation Status -From `PyPI `_ +From `PyPI `_:: pip install gitdb SPEEDUPS ======== -If you want to go up to 20% faster, you can install gitdb-speedups with: +If you want to go up to 20% faster, you can install gitdb-speedups with:: pip install gitdb-speedups +However, please note that gitdb-speedups is not currently maintained. + REQUIREMENTS ============ +* smmap - declared as a dependency, automatically installed * pytest - for running the tests SOURCE ====== -The source is available in a git repository at gitorious and github: + +The source is available in a git repository on GitHub: https://github.com/gitpython-developers/gitdb -Once the clone is complete, please be sure to initialize the submodules using +Once the clone is complete, please be sure to initialize the submodule using:: cd gitdb git submodule update --init -Run the tests with +Run the tests with:: pytest @@ -53,13 +57,13 @@ DEVELOPMENT .. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg :target: https://github.com/gitpython-developers/gitdb/actions -The library is considered mature, and not under active development. It's primary (known) use is in git-python. +The library is considered mature, and not under active development. Its primary (known) use is in GitPython. INFRASTRUCTURE ============== -* Mailing List - * http://groups.google.com/group/git-python +* Discussions + * https://github.com/gitpython-developers/GitPython/discussions * Issue Tracker * https://github.com/gitpython-developers/gitdb/issues From e998429c01f928da7ff7c922ba3f1249c43ff569 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 07:47:10 -0400 Subject: [PATCH 0790/1790] Set Dependabot submodule update cadence to weekly This changes it from monthly to weekly. See #99 and https://github.com/gitpython-developers/GitPython/pull/1702#issuecomment-1761182333 for context. --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5acde1a9a..2fe73ca77 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,4 +8,4 @@ updates: - package-ecosystem: "gitsubmodule" directory: "/" schedule: - interval: "monthly" + interval: "weekly" From a9dab913ae6371b645e7509fb21257ee56124063 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 07:53:39 -0400 Subject: [PATCH 0791/1790] Set submodule update cadence to weekly This sets the Dependabot submodule update cadence from montly to weekly, as requested in: https://github.com/gitpython-developers/GitPython/pull/1702#issuecomment-1761182333 (This change in GitPython corresponds directly to https://github.com/gitpython-developers/gitdb/pull/104 in gitdb.) --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5acde1a9a..2fe73ca77 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,4 +8,4 @@ updates: - package-ecosystem: "gitsubmodule" directory: "/" schedule: - interval: "monthly" + interval: "weekly" From 00aee49d2027bdf9c0f1c8c2065aa1b67b5ecb2e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 05:32:01 -0400 Subject: [PATCH 0792/1790] Never modify sys.path This removes the logic that, under some (most) circumstances when not using a version from PyPI, would insert the location of the gitdb git-submodule near the front of sys.path. (As noted in #1717, the specific way this was being done was not causing the git-submodule's version of gitdb to actually be used. But it was still modifying sys.path, which this now prevents.) The installation test, which had verified the insertion into sys.path, is modified accordingly, except that for now the check that the very first sys.path entry is undisturbed is kept in place. --- git/__init__.py | 29 ----------------------------- test/test_installation.py | 24 ++++++++++++++++++++++-- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 05a02a7ff..5b55e59eb 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -6,38 +6,11 @@ # flake8: noqa # @PydevCodeAnalysisIgnore from git.exc import * # @NoMove @IgnorePep8 -import os -import os.path as osp -import sys - from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING from git.types import PathLike __version__ = "git" - -# { Initialization -def _init_externals() -> None: - """Initialize external projects by putting them into the path""" - if __version__ == "git" and "PYOXIDIZER" not in os.environ: - sys.path.insert(1, osp.join(osp.dirname(__file__), "ext", "gitdb")) - - try: - import gitdb - except ImportError as e: - raise ImportError("'gitdb' could not be found in your PYTHONPATH") from e - # END verify import - - -# } END initialization - - -################# -_init_externals() -################# - -# { Imports - from gitdb.util import to_hex_sha try: @@ -62,8 +35,6 @@ def _init_externals() -> None: except GitError as _exc: raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc -# } END imports - # __all__ must be statically defined by py.typed support # __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] __all__ = [ diff --git a/test/test_installation.py b/test/test_installation.py index 0cb0c71fa..3a0e43973 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -25,6 +25,7 @@ def setUp_venv(self, rw_dir): @with_rw_directory def test_installation(self, rw_dir): self.setUp_venv(rw_dir) + result = subprocess.run( [self.pip, "install", "."], stdout=subprocess.PIPE, @@ -35,12 +36,32 @@ def test_installation(self, rw_dir): result.returncode, msg=result.stderr or result.stdout or "Can't install project", ) - result = subprocess.run([self.python, "-c", "import git"], stdout=subprocess.PIPE, cwd=self.sources) + + result = subprocess.run( + [self.python, "-c", "import git"], + stdout=subprocess.PIPE, + cwd=self.sources, + ) self.assertEqual( 0, result.returncode, msg=result.stderr or result.stdout or "Selftest failed", ) + + result = subprocess.run( + [self.python, "-c", "import gitdb; import smmap"], + stdout=subprocess.PIPE, + cwd=self.sources, + ) + self.assertEqual( + 0, + result.returncode, + msg=result.stderr or result.stdout or "Dependencies not installed", + ) + + # Even IF gitdb or any other dependency is supplied during development + # by inserting its location into PYTHONPATH or otherwise patched into + # sys.path, make sure it is not wrongly inserted as the *first* entry. result = subprocess.run( [self.python, "-c", "import sys;import git; print(sys.path)"], stdout=subprocess.PIPE, @@ -53,4 +74,3 @@ def test_installation(self, rw_dir): syspath[0], msg="Failed to follow the conventions for https://docs.python.org/3/library/sys.html#sys.path", ) - self.assertTrue(syspath[1].endswith("gitdb"), msg="Failed to add gitdb to sys.path") From 610c46ddf8c535d8a7bd75921fa8a215648f94ec Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 05:36:06 -0400 Subject: [PATCH 0793/1790] Document how to use vendored dependencies This adds a subsection to the end of the installation instructions in the readme to explain how to cause the versions of the gitdb and/or smmap dependencies that are vendored as submodules of the GitPython repository to be used, instead of the PyPI versions, in the infrequent case that this is desired. This goes along with he removal of the logic that conditionally modified sys.path, since that logic was intended to facilitate this (and at one time had). The approach now documented in the readme uses editable installs and does not involve modifying sys.path. Because the need for this is uncommon, it may end up being moved entirely into documentation in the doc/ directory in the future. --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index a7942fd2f..889ea635f 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,20 @@ pip install -e ".[test]" In the less common case that you do not want to install test dependencies, `pip install -e .` can be used instead. +#### With editable *dependencies* (not preferred, and rarely needed) + +In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediatley reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. + +If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: + +```bash +pip install -e ".[test]" -e git/ext/gitdb -e git/ext/gitdb/gitdb/ext/smmap +``` + +The submodules must have been cloned for that to work, but that will already be the case if you have run `./init-tests-after-clone.sh`. You can use `pip list` to check which packages are installed editably and which are installed normally. + +To reiterate, this approach should only rarely be used. For most development it is preferable to allow the gitdb and smmap dependencices to be retrieved automatically from PyPI in their latest stable packaged versions. + ### Limitations #### Leakage of System Resources From 88e06d715dd88074155e35f1761d6a554c4edd9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Oct 2023 15:29:25 +0000 Subject: [PATCH 0794/1790] Bump git/ext/gitdb from `8ec2390` to `ec58b7e` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `8ec2390` to `ec58b7e`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/8ec23904f44fbbb9bb560b83bd891a9b48be7645...ec58b7e55c8fec2479290a22faa293799f6805fc) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 8ec23904f..ec58b7e55 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 8ec23904f44fbbb9bb560b83bd891a9b48be7645 +Subproject commit ec58b7e55c8fec2479290a22faa293799f6805fc From 7dd209593d08067e005c92b6856a07dc5a2b618a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 10 Oct 2023 01:08:54 -0400 Subject: [PATCH 0795/1790] Fix docstrings that intend '\' literally This intended '\' literally, but it was actually '' because the \' became just ' (backslash-apostrophe becomes just aspostrophe). --- git/objects/base.py | 2 +- git/util.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index 1d07fd0f6..34279c639 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -212,7 +212,7 @@ def name(self) -> str: @property def abspath(self) -> PathLike: - """ + R""" :return: Absolute path to this index object in the file system ( as opposed to the .path field which is a path relative to the git repository ). diff --git a/git/util.py b/git/util.py index 97f461a83..ccadb5881 100644 --- a/git/util.py +++ b/git/util.py @@ -239,7 +239,7 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: - """Join path tokens together similar to osp.join, but always use + R"""Join path tokens together similar to osp.join, but always use '/' instead of possibly '\' on windows.""" path = str(a) for b in p: @@ -277,7 +277,7 @@ def to_native_path_linux(path: PathLike) -> str: def join_path_native(a: PathLike, *p: PathLike) -> PathLike: - """ + R""" As join path, but makes sure an OS native path is returned. This is only needed to play it safe on my dear windows and to assure nice paths that only use '\'""" From ffcbf07e7c879bfeb031b41728fdf2ecfa63f66e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 10 Oct 2023 01:29:34 -0400 Subject: [PATCH 0796/1790] Put all regex patterns in r-strings This improves consistency, because most were already. For a few it allows backslashes to removed, improving readability. Even for the others, some editors will highlight them as regular expressions now that they are raw string literals. --- git/cmd.py | 2 +- git/diff.py | 2 +- git/refs/log.py | 2 +- git/repo/base.py | 4 ++-- test/test_exc.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 7c448e3f2..e18bff8c1 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -264,7 +264,7 @@ class Git(LazyMixin): _excluded_ = ("cat_file_all", "cat_file_header", "_version_info") - re_unsafe_protocol = re.compile("(.+)::.+") + re_unsafe_protocol = re.compile(r"(.+)::.+") def __getstate__(self) -> Dict[str, Any]: return slots_to_dict(self, exclude=self._excluded_) diff --git a/git/diff.py b/git/diff.py index 3e3de7bc1..843d96152 100644 --- a/git/diff.py +++ b/git/diff.py @@ -53,7 +53,7 @@ # Special object to compare against the empty tree in diffs NULL_TREE = object() -_octal_byte_re = re.compile(b"\\\\([0-9]{3})") +_octal_byte_re = re.compile(rb"\\([0-9]{3})") def _octal_repl(matchobj: Match) -> bytes: diff --git a/git/refs/log.py b/git/refs/log.py index ef3f86b8b..9b02051d3 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -41,7 +41,7 @@ class RefLogEntry(Tuple[str, str, Actor, Tuple[int, int], str]): """Named tuple allowing easy access to the revlog data fields""" - _re_hexsha_only = re.compile("^[0-9A-Fa-f]{40}$") + _re_hexsha_only = re.compile(r"^[0-9A-Fa-f]{40}$") __slots__ = () def __repr__(self) -> str: diff --git a/git/repo/base.py b/git/repo/base.py index bc1b8876d..23136a1d1 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -131,8 +131,8 @@ class Repo(object): # precompiled regex re_whitespace = re.compile(r"\s+") - re_hexsha_only = re.compile("^[0-9A-Fa-f]{40}$") - re_hexsha_shortened = re.compile("^[0-9A-Fa-f]{4,40}$") + re_hexsha_only = re.compile(r"^[0-9A-Fa-f]{40}$") + re_hexsha_shortened = re.compile(r"^[0-9A-Fa-f]{4,40}$") re_envvars = re.compile(r"(\$(\{\s?)?[a-zA-Z_]\w*(\}\s?)?|%\s?[a-zA-Z_]\w*\s?%)") re_author_committer_start = re.compile(r"^(author|committer)") re_tab_full_line = re.compile(r"^\t(.*)$") diff --git a/test/test_exc.py b/test/test_exc.py index 9e125d246..0c3ff35a8 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -102,7 +102,7 @@ def test_CommandError_unicode(self, case): if subs is not None: # Substrings (must) already contain opening `'`. - subs = "(? Date: Tue, 10 Oct 2023 01:44:25 -0400 Subject: [PATCH 0797/1790] Remove r prefix from strings that need not be raw These are a couple strings that, in addition to not having any escape sequences, don't represent regular expressions, Windows paths, or anything else that would be clarified by raw literals. --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 54f1f4723..d19b0c0ec 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -170,7 +170,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ("index", "GitPython.tex", r"GitPython Documentation", r"Michael Trier", "manual"), + ("index", "GitPython.tex", "GitPython Documentation", "Michael Trier", "manual"), ] # The name of an image file (relative to this directory) to place at the top of From 35fd65bda145eb946ba1f69ff7ed371424f1a0a6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Oct 2023 16:37:12 -0400 Subject: [PATCH 0798/1790] Fix _index_from_*_format docstrings (proc reading) In the git.diff.Diff class, the _index_from_patch_format and _index_from_raw_format methods' docstrings had still described them as reading from streams, even though they have instead read from processes (and taken "proc", not "stream", arguments) since #519 when the change was made in a5db3d3 to fix a freezing bug. This updates the docstrings to reflect that they read from processes. --- git/diff.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/git/diff.py b/git/diff.py index 843d96152..8ee2527d4 100644 --- a/git/diff.py +++ b/git/diff.py @@ -486,10 +486,12 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m @classmethod def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex: - """Create a new DiffIndex from the given text which must be in patch format - :param repo: is the repository we are operating on - it is required - :param stream: result of 'git diff' as a stream (supporting file protocol) - :return: git.DiffIndex""" + """Create a new DiffIndex from the given process output which must be in patch format. + + :param repo: The repository we are operating on + :param proc: ``git diff`` process to read from (supports :class:`Git.AutoInterrupt` wrapper) + :return: git.DiffIndex + """ ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text_list: List[bytes] = [] @@ -644,8 +646,12 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non @classmethod def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex": - """Create a new DiffIndex from the given stream which must be in raw format. - :return: git.DiffIndex""" + """Create a new DiffIndex from the given process output which must be in raw format. + + :param repo: The repository we are operating on + :param proc: Process to read output from + :return: git.DiffIndex + """ # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore From 5af74467ba536dcd0e22c49345eb791eb48cbf45 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Oct 2023 23:24:03 -0400 Subject: [PATCH 0799/1790] Fix case in IndexObject.abspath exception message The message refers to the (public) working_tree_dir attribute by name, so that should be uncapitalized to reflect the case by which it must be accessed, even when it appears at the beginning of a sentence. --- git/objects/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/base.py b/git/objects/base.py index 34279c639..0dab5ccdb 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -221,4 +221,4 @@ def abspath(self) -> PathLike: if self.repo.working_tree_dir is not None: return join_path_native(self.repo.working_tree_dir, self.path) else: - raise WorkTreeRepositoryUnsupported("Working_tree_dir was None or empty") + raise WorkTreeRepositoryUnsupported("working_tree_dir was None or empty") From 692e59e9060f6cfc4cd49d8e83860522666fe966 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 16 Oct 2023 01:53:02 -0400 Subject: [PATCH 0800/1790] Remove Commit._deserialize doc for param_from_rev_list The git.objects.commit.Commit._deserialize method stopped accepting a param_from_rev_list argument in ae5a69f, but the documentation for that parameter was never removed. Because that was the only part of the method's docstring, and it is a nonpublic method, and the associated _serialize method does not have a docstring, this change simply removes the _deserialize method's docstring without adding anything. --- git/objects/commit.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 88c485d09..fd65fa1e4 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -688,10 +688,6 @@ def _serialize(self, stream: BytesIO) -> "Commit": return self def _deserialize(self, stream: BytesIO) -> "Commit": - """ - :param from_rev_list: if true, the stream format is coming from the rev-list command - Otherwise it is assumed to be a plain data stream from our object - """ readline = stream.readline self.tree = Tree(self.repo, hex_to_bin(readline().split()[1]), Tree.tree_id << 12, "") From ab46192e88feebd6ff8e9ffbec5142e47ff03207 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 17 Oct 2023 16:28:20 -0400 Subject: [PATCH 0801/1790] Add missing space in Submodule.update debug message --- git/objects/submodule/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 473ebde76..2a340399d 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -755,7 +755,7 @@ def update( if len(base_commit) == 0 or (base_commit[0] is not None and base_commit[0].hexsha == hexsha): if force: msg = "Will force checkout or reset on local branch that is possibly in the future of" - msg += "the commit it will be checked out to, effectively 'forgetting' new commits" + msg += " the commit it will be checked out to, effectively 'forgetting' new commits" log.debug(msg) else: msg = "Skipping %s on branch '%s' of submodule repo '%s' as it contains un-pushed commits" From 1114828de8e7cedaa62776522e0284ae2fe8b5d0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 17 Oct 2023 20:28:31 -0400 Subject: [PATCH 0802/1790] Remove obsolete comment in Submodule.module The local (i.e. late) import was removed in 7b3ef45, but the comment about it on (what was) the preceding line has persisted until now. --- git/objects/submodule/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 2a340399d..e1946c27a 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1217,7 +1217,6 @@ def module(self) -> "Repo": """:return: Repo instance initialized from the repository at our submodule path :raise InvalidGitRepositoryError: if a repository was not available. This could also mean that it was not yet initialized""" - # late import to workaround circular dependencies module_checkout_abspath = self.abspath try: repo = git.Repo(module_checkout_abspath) From f95d4fd2e4d7bc5802633ef0c6f6abb818e1559c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 17 Oct 2023 21:31:43 -0400 Subject: [PATCH 0803/1790] Clarify "master repository" in RootModule docs This changes master repository to superproject (master repository) in the RootModule class docstring and in the tutorial, to make even clearer what this is referring to. This way, users who are less familiar with submodules will be less likely to confuse this with a "master" branch (since "master" is one of the popular default branch names), while users who are more familiar with submodules and may search for the official term "superproject" will find the docs when doing so. This retains "master repository" parenthesized rather than completely replacing it because although "superproject" is the official term for this, it is a bit obscure and unintuitive. --- doc/source/tutorial.rst | 3 +-- git/objects/submodule/root.py | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index fcbc18bff..fd3b14c57 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -413,7 +413,7 @@ If you obtained your submodule object by traversing a tree object which is not r you have to inform the submodule about its actual commit to retrieve the data from by using the ``set_parent_commit(...)`` method. -The special :class:`RootModule ` type allows you to treat your master repository as root of a hierarchy of submodules, which allows very convenient submodule handling. Its ``update(...)`` method is reimplemented to provide an advanced way of updating submodules as they change their values over time. The update method will track changes and make sure your working tree and submodule checkouts stay consistent, which is very useful in case submodules get deleted or added to name just two of the handled cases. +The special :class:`RootModule ` type allows you to treat your superproject (master repository) as root of a hierarchy of submodules, which allows very convenient submodule handling. Its ``update(...)`` method is reimplemented to provide an advanced way of updating submodules as they change their values over time. The update method will track changes and make sure your working tree and submodule checkouts stay consistent, which is very useful in case submodules get deleted or added to name just two of the handled cases. Additionally, GitPython adds functionality to track a specific branch, instead of just a commit. Supported by customized update methods, you are able to automatically update submodules to the latest revision available in the remote repository, as well as to keep track of changes and movements of these submodules. To use it, set the name of the branch you want to track to the ``submodule.$name.branch`` option of the *.gitmodules* file, and use GitPython update methods on the resulting repository with the ``to_latest_revision`` parameter turned on. In the latter case, the sha of your submodule will be ignored, instead a local tracking branch will be updated to the respective remote branch automatically, provided there are no local changes. The resulting behaviour is much like the one of svn::externals, which can be useful in times. @@ -545,4 +545,3 @@ And even more ... There is more functionality in there, like the ability to archive repositories, get stats and logs, blame, and probably a few other things that were not mentioned here. Check the unit tests for an in-depth introduction on how each function is supposed to be used. - diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 0cbc262ca..4f4906c51 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -43,9 +43,11 @@ class RootUpdateProgress(UpdateProgress): class RootModule(Submodule): + """A (virtual) root of all submodules in the given repository. - """A (virtual) Root of all submodules in the given repository. It can be used - to more easily traverse all submodules of the master repository""" + This can be used to more easily traverse all submodules of the + superproject (master repository). + """ __slots__ = () From 9fea4881791a116a7f1edd49d7c16b4218dc991f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 17 Oct 2023 21:52:13 -0400 Subject: [PATCH 0804/1790] Change spelling from "commit'ish" to "commit-ish" Since the hyphen spelling is what the official Git docs use. This change affects only docstrings. --- git/index/base.py | 2 +- git/objects/submodule/base.py | 2 +- git/objects/submodule/root.py | 2 +- git/refs/symbolic.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 0cdeb1ce5..5da904b4e 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1300,7 +1300,7 @@ def reset( but if True, this method behaves like HEAD.reset. :param paths: if given as an iterable of absolute or repository-relative paths, - only these will be reset to their state at the given commit'ish. + only these will be reset to their state at the given commit-ish. The paths need to exist at the commit, otherwise an exception will be raised. diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index e1946c27a..61c300652 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1095,7 +1095,7 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) contain the .gitmodules blob. :param commit: - Commit'ish reference pointing at the root_tree, or None to always point to the + Commit-ish reference pointing at the root_tree, or None to always point to the most recent commit :param check: if True, relatively expensive checks will be performed to verify diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 4f4906c51..d338441ef 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -91,7 +91,7 @@ def update( Additionally it will detect addition and removal of submodules, which will be handled gracefully. - :param previous_commit: If set to a commit'ish, the commit we should use + :param previous_commit: If set to a commit-ish, the commit we should use as the previous commit the HEAD pointed to before it was set to the commit it points to now. If None, it defaults to HEAD@{1} otherwise :param recursive: if True, the children of submodules will be updated as well diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 549160444..7cc304136 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -641,7 +641,7 @@ def create( :param reference: The reference to which the new symbolic reference should point to. - If it is a commit'ish, the symbolic ref will be detached. + If it is a commit-ish, the symbolic ref will be detached. :param force: if True, force creation even if a symbolic reference with that name already exists. From 4536b634b7e5d74867d56ca413d27ed81fca0e1b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 17 Oct 2023 23:17:48 -0400 Subject: [PATCH 0805/1790] Update git-source citation in Reference.set_object Reference.set_object explains its handling of oldbinsha by quoting a comment from refs.c in the Git source code. However, that comment now appears in refs/files-backend.c in that codebase. This updates the reference so readers can look it up and find the comment in its surrounding context. The commit to the git project's source code that moved the code that includes that comment is: https://github.com/git/git/commit/7bd9bcf372d4c03bb7034346d72ae1318e2d0742 --- git/refs/reference.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/git/refs/reference.py b/git/refs/reference.py index 4f9e3a0a7..dea1af68c 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -86,18 +86,19 @@ def set_object( super(Reference, self).set_object(object, logmsg) if oldbinsha is not None: - # /* from refs.c in git-source - # * Special hack: If a branch is updated directly and HEAD - # * points to it (may happen on the remote side of a push - # * for example) then logically the HEAD reflog should be - # * updated too. - # * A generic solution implies reverse symref information, - # * but finding all symrefs pointing to the given branch - # * would be rather costly for this rare event (the direct - # * update of a branch) to be worth it. So let's cheat and - # * check with HEAD only which should cover 99% of all usage - # * scenarios (even 100% of the default ones). - # */ + # From refs/files-backend.c in git-source: + # /* + # * Special hack: If a branch is updated directly and HEAD + # * points to it (may happen on the remote side of a push + # * for example) then logically the HEAD reflog should be + # * updated too. + # * A generic solution implies reverse symref information, + # * but finding all symrefs pointing to the given branch + # * would be rather costly for this rare event (the direct + # * update of a branch) to be worth it. So let's cheat and + # * check with HEAD only which should cover 99% of all usage + # * scenarios (even 100% of the default ones). + # */ self.repo.head.log_append(oldbinsha, logmsg) # END check if the head From 59d208c0e9566b02a835215bbbeddcce37da0afd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 00:43:55 -0400 Subject: [PATCH 0806/1790] Fix message in SymbolicReference.from_path This rarely-seen ValueError message had said SymbolicRef, and is now changed to SymbolicReference. --- git/refs/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 7cc304136..45496c9d9 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -798,7 +798,7 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere instance: T_References instance = ref_type(repo, path) if instance.__class__ == SymbolicReference and instance.is_detached: - raise ValueError("SymbolRef was detached, we drop it") + raise ValueError("SymbolicRef was detached, we drop it") else: return instance From add46d9daa7664628ba3512297a2401981694c98 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 00:45:32 -0400 Subject: [PATCH 0807/1790] Use "is" to compare __class__ Since the value of __class__ is a type, comparing it to another type object should use "is" rather than "==". Some of these, involving type(), were fixed in bf7af69, but flake8 did not catch the .__class__ variation addressed here. --- git/refs/symbolic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 45496c9d9..5ff84367d 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -769,7 +769,7 @@ def iter_items( List is lexicographically sorted The returned objects represent actual subclasses, such as Head or TagReference""" - return (r for r in cls._iter_items(repo, common_path) if r.__class__ == SymbolicReference or not r.is_detached) + return (r for r in cls._iter_items(repo, common_path) if r.__class__ is SymbolicReference or not r.is_detached) @classmethod def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_References: @@ -797,7 +797,7 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere try: instance: T_References instance = ref_type(repo, path) - if instance.__class__ == SymbolicReference and instance.is_detached: + if instance.__class__ is SymbolicReference and instance.is_detached: raise ValueError("SymbolicRef was detached, we drop it") else: return instance From cd16a3534eedb5a0990293e16a6371ca16d8776a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 04:21:41 -0400 Subject: [PATCH 0808/1790] Revise docstrings and comments for clarity and formatting In the git module (including the modules it contains). This also makes one small change in doc/ to synchronize with a change made in a docstring. --- git/__init__.py | 2 + git/cmd.py | 357 +++++++++++---------- git/compat.py | 12 +- git/config.py | 225 +++++++------ git/db.py | 23 +- git/diff.py | 158 ++++----- git/exc.py | 36 ++- git/index/__init__.py | 4 +- git/index/base.py | 389 +++++++++++----------- git/index/fun.py | 146 +++++---- git/index/typ.py | 24 +- git/index/util.py | 16 +- git/objects/__init__.py | 11 +- git/objects/base.py | 75 +++-- git/objects/blob.py | 10 +- git/objects/commit.py | 171 +++++----- git/objects/fun.py | 95 +++--- git/objects/submodule/__init__.py | 4 +- git/objects/submodule/base.py | 513 ++++++++++++++++-------------- git/objects/submodule/root.py | 142 +++++---- git/objects/submodule/util.py | 23 +- git/objects/tag.py | 32 +- git/objects/tree.py | 89 +++--- git/objects/util.py | 157 +++++---- git/refs/__init__.py | 2 +- git/refs/head.py | 75 +++-- git/refs/log.py | 111 ++++--- git/refs/reference.py | 52 +-- git/refs/remote.py | 16 +- git/refs/symbolic.py | 294 ++++++++++------- git/refs/tag.py | 48 +-- git/remote.py | 300 +++++++++-------- git/repo/__init__.py | 4 +- git/repo/base.py | 426 +++++++++++++++---------- git/repo/fun.py | 4 +- git/types.py | 3 + git/util.py | 213 +++++++------ 37 files changed, 2370 insertions(+), 1892 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 5b55e59eb..46d54a960 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -3,8 +3,10 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + # flake8: noqa # @PydevCodeAnalysisIgnore + from git.exc import * # @NoMove @IgnorePep8 from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING from git.types import PathLike diff --git a/git/cmd.py b/git/cmd.py index e18bff8c1..f8b6c9e78 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -3,7 +3,9 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + from __future__ import annotations + import re import contextlib import io @@ -103,20 +105,21 @@ def handle_process_output( decode_streams: bool = True, kill_after_timeout: Union[None, float] = None, ) -> None: - """Registers for notifications to learn that process output is ready to read, and dispatches lines to - the respective line handlers. + """Register for notifications to learn that process output is ready to read, and + dispatch lines to the respective line handlers. + This function returns once the finalizer returns. - :return: result of finalizer - :param process: subprocess.Popen instance + :return: Result of finalizer + :param process: :class:`subprocess.Popen` instance :param stdout_handler: f(stdout_line_string), or None :param stderr_handler: f(stderr_line_string), or None :param finalizer: f(proc) - wait for proc to finish :param decode_streams: - Assume stdout/stderr streams are binary and decode them before pushing \ + Assume stdout/stderr streams are binary and decode them before pushing their contents to handlers. - Set it to False if `universal_newline == True` (then streams are in text-mode) - or if decoding must happen later (i.e. for Diffs). + Set it to False if ``universal_newlines == True`` (then streams are in + text mode) or if decoding must happen later (i.e. for Diffs). :param kill_after_timeout: float or None, Default = None To specify a timeout in seconds for the git command, after which the process @@ -232,13 +235,11 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc # see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal PROC_CREATIONFLAGS = ( CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP if is_win else 0 # type: ignore[attr-defined] -) # mypy error if not windows +) # mypy error if not Windows. class Git(LazyMixin): - - """ - The Git class manages communication with the Git binary. + """The Git class manages communication with the Git binary. It provides a convenient interface to calling the Git binary, such as in:: @@ -274,44 +275,41 @@ def __setstate__(self, d: Dict[str, Any]) -> None: # CONFIGURATION - git_exec_name = "git" # default that should work on linux and windows + git_exec_name = "git" # Default that should work on Linux and Windows. - # Enables debugging of GitPython's git commands + # Enables debugging of GitPython's git commands. GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) # If True, a shell will be used when executing git commands. # This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 # and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. - # Override this value using `Git.USE_SHELL = True` + # Override this value using `Git.USE_SHELL = True`. USE_SHELL = False - # Provide the full path to the git executable. Otherwise it assumes git is in the path + # Provide the full path to the git executable. Otherwise it assumes git is in the path. _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" GIT_PYTHON_GIT_EXECUTABLE = None - # note that the git executable is actually found during the refresh step in - # the top level __init__ + # Note that the git executable is actually found during the refresh step in + # the top level __init__. @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: """This gets called by the refresh function (see the top level __init__).""" - # discern which path to refresh with + # Discern which path to refresh with. if path is not None: new_git = os.path.expanduser(path) new_git = os.path.abspath(new_git) else: new_git = os.environ.get(cls._git_exec_env_var, cls.git_exec_name) - # keep track of the old and new git executable path + # Keep track of the old and new git executable path. old_git = cls.GIT_PYTHON_GIT_EXECUTABLE cls.GIT_PYTHON_GIT_EXECUTABLE = new_git - # test if the new git executable path is valid - - # - a GitCommandNotFound error is spawned by ourselves - # - a PermissionError is spawned if the git executable provided - # cannot be executed for whatever reason - + # Test if the new git executable path is valid. A GitCommandNotFound error is + # spawned by us. A PermissionError is spawned if the git executable cannot be + # executed for whatever reason. has_git = False try: cls().version() @@ -319,7 +317,7 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: except (GitCommandNotFound, PermissionError): pass - # warn or raise exception if test failed + # Warn or raise exception if test failed. if not has_git: err = ( dedent( @@ -334,18 +332,18 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: % cls._git_exec_env_var ) - # revert to whatever the old_git was + # Revert to whatever the old_git was. cls.GIT_PYTHON_GIT_EXECUTABLE = old_git if old_git is None: - # on the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is - # None) we only are quiet, warn, or error depending on the - # GIT_PYTHON_REFRESH value - - # determine what the user wants to happen during the initial - # refresh we expect GIT_PYTHON_REFRESH to either be unset or - # be one of the following values: - # 0|q|quiet|s|silence + # On the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is None) we only + # are quiet, warn, or error depending on the GIT_PYTHON_REFRESH value. + + # Determine what the user wants to happen during the initial refresh we + # expect GIT_PYTHON_REFRESH to either be unset or be one of the + # following values: + # + # 0|q|quiet|s|silence|n|none # 1|w|warn|warning # 2|r|raise|e|error @@ -410,14 +408,13 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: ) raise ImportError(err) - # we get here if this was the init refresh and the refresh mode - # was not error, go ahead and set the GIT_PYTHON_GIT_EXECUTABLE - # such that we discern the difference between a first import - # and a second import + # We get here if this was the init refresh and the refresh mode was not + # error. Go ahead and set the GIT_PYTHON_GIT_EXECUTABLE such that we + # discern the difference between a first import and a second import. cls.GIT_PYTHON_GIT_EXECUTABLE = cls.git_exec_name else: - # after the first refresh (when GIT_PYTHON_GIT_EXECUTABLE - # is no longer None) we raise an exception + # After the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is no longer + # None) we raise an exception. raise GitCommandNotFound("git", err) return has_git @@ -438,18 +435,18 @@ def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: @classmethod def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: + """Remove any backslashes from urls to be written in config files. + + Windows might create config files containing paths with backslashes, + but git stops liking them as it will escape the backslashes. Hence we + undo the escaping just to be sure. + """ if is_cygwin is None: is_cygwin = cls.is_cygwin() if is_cygwin: url = cygpath(url) else: - """Remove any backslashes from urls to be written in config files. - - Windows might create config files containing paths with backslashes, - but git stops liking them as it will escape the backslashes. - Hence we undo the escaping just to be sure. - """ url = os.path.expandvars(url) if url.startswith("~"): url = os.path.expanduser(url) @@ -458,12 +455,11 @@ def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: @classmethod def check_unsafe_protocols(cls, url: str) -> None: - """ - Check for unsafe protocols. + """Check for unsafe protocols. Apart from the usual protocols (http, git, ssh), - Git allows "remote helpers" that have the form ``::
``, - one of these helpers (``ext::``) can be used to invoke any arbitrary command. + Git allows "remote helpers" that have the form ``::
``. + One of these helpers (``ext::``) can be used to invoke any arbitrary command. See: @@ -479,8 +475,7 @@ def check_unsafe_protocols(cls, url: str) -> None: @classmethod def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: - """ - Check for unsafe options. + """Check for unsafe options. Some options that are passed to `git ` can be used to execute arbitrary commands, this are blocked by default. @@ -496,17 +491,21 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> ) class AutoInterrupt(object): - """Kill/Interrupt the stored process instance once this instance goes out of scope. It is - used to prevent processes piling up in case iterators stop reading. - Besides all attributes are wired through to the contained process object. + """Process wrapper that terminates the wrapped process on finalization. + + This kills/interrupts the stored process instance once this instance goes out of + scope. It is used to prevent processes piling up in case iterators stop reading. - The wait method was overridden to perform automatic status code checking - and possibly raise.""" + All attributes are wired through to the contained process object. + + The wait method is overridden to perform automatic status code checking and + possibly raise. + """ __slots__ = ("proc", "args", "status") # If this is non-zero it will override any status code during - # _terminate, used to prevent race conditions in testing + # _terminate, used to prevent race conditions in testing. _status_code_if_terminate: int = 0 def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: @@ -527,7 +526,7 @@ def _terminate(self) -> None: proc.stdout.close() if proc.stderr: proc.stderr.close() - # did the process finish already so we have a return code ? + # Did the process finish already so we have a return code? try: if proc.poll() is not None: self.status = self._status_code_if_terminate or proc.poll() @@ -535,23 +534,23 @@ def _terminate(self) -> None: except OSError as ex: log.info("Ignored error after process had died: %r", ex) - # can be that nothing really exists anymore ... + # It can be that nothing really exists anymore... if os is None or getattr(os, "kill", None) is None: return None - # try to kill it + # Try to kill it. try: proc.terminate() - status = proc.wait() # ensure process goes away + status = proc.wait() # Ensure the process goes away. self.status = self._status_code_if_terminate or status except OSError as ex: log.info("Ignored error after process had died: %r", ex) except AttributeError: - # try windows - # for some reason, providing None for stdout/stderr still prints something. This is why - # we simply use the shell and redirect to nul. Its slower than CreateProcess, question - # is whether we really want to see all these messages. Its annoying no matter what. + # Try Windows. + # For some reason, providing None for stdout/stderr still prints something. This is why + # we simply use the shell and redirect to nul. Slower than CreateProcess. The question + # is whether we really want to see all these messages. It's annoying no matter what. if is_win: call( ("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), @@ -571,7 +570,8 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: :param stderr: Previously read value of stderr, in case stderr is already closed. :warn: May deadlock if output or error pipes are used and not handled separately. - :raise GitCommandError: if the return status is not 0""" + :raise GitCommandError: If the return status is not 0. + """ if stderr is None: stderr_b = b"" stderr_b = force_bytes(data=stderr, encoding="utf-8") @@ -579,7 +579,7 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: if self.proc is not None: status = self.proc.wait() p_stderr = self.proc.stderr - else: # Assume the underlying proc was killed earlier or never existed + else: # Assume the underlying proc was killed earlier or never existed. status = self.status p_stderr = None @@ -605,19 +605,22 @@ def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> byte class CatFileContentStream(object): """Object representing a sized read-only stream returning the contents of an object. - It behaves like a stream, but counts the data read and simulates an empty + + This behaves like a stream, but counts the data read and simulates an empty stream once our sized content region is empty. - If not all data is read to the end of the object's lifetime, we read the - rest to assure the underlying stream continues to work.""" + + If not all data are read to the end of the object's lifetime, we read the + rest to assure the underlying stream continues to work. + """ __slots__: Tuple[str, ...] = ("_stream", "_nbr", "_size") def __init__(self, size: int, stream: IO[bytes]) -> None: self._stream = stream self._size = size - self._nbr = 0 # num bytes read + self._nbr = 0 # Number of bytes read. - # special case: if the object is empty, has null bytes, get the + # Special case: If the object is empty, has null bytes, get the # final newline right away. if size == 0: stream.read(1) @@ -628,16 +631,16 @@ def read(self, size: int = -1) -> bytes: if bytes_left == 0: return b"" if size > -1: - # assure we don't try to read past our limit + # Ensure we don't try to read past our limit. size = min(bytes_left, size) else: - # they try to read all, make sure its not more than what remains + # They try to read all, make sure it's not more than what remains. size = bytes_left # END check early depletion data = self._stream.read(size) self._nbr += len(data) - # check for depletion, read our final byte to make the stream usable by others + # Check for depletion, read our final byte to make the stream usable by others. if self._size - self._nbr == 0: self._stream.read(1) # final newline # END finish reading @@ -647,7 +650,7 @@ def readline(self, size: int = -1) -> bytes: if self._nbr == self._size: return b"" - # clamp size to lowest allowed value + # Clamp size to lowest allowed value. bytes_left = self._size - self._nbr if size > -1: size = min(bytes_left, size) @@ -658,7 +661,7 @@ def readline(self, size: int = -1) -> bytes: data = self._stream.readline(size) self._nbr += len(data) - # handle final byte + # Handle final byte. if self._size - self._nbr == 0: self._stream.read(1) # END finish reading @@ -669,7 +672,7 @@ def readlines(self, size: int = -1) -> List[bytes]: if self._nbr == self._size: return [] - # leave all additional logic to our readline method, we just check the size + # Leave all additional logic to our readline method, we just check the size. out = [] nbr = 0 while True: @@ -701,8 +704,8 @@ def __next__(self) -> bytes: def __del__(self) -> None: bytes_left = self._size - self._nbr if bytes_left: - # read and discard - seeking is impossible within a stream - # includes terminating newline + # Read and discard - seeking is impossible within a stream. + # This includes any terminating newline. self._stream.read(bytes_left + 1) # END handle incomplete read @@ -711,9 +714,10 @@ def __init__(self, working_dir: Union[None, PathLike] = None): :param working_dir: Git directory we should work in. If None, we always work in the current - directory as returned by os.getcwd(). + directory as returned by :func:`os.getcwd`. It is meant to be the working tree directory if available, or the - .git directory in case of bare repositories.""" + ``.git`` directory in case of bare repositories. + """ super(Git, self).__init__() self._working_dir = expand_path(working_dir) self._git_options: Union[List[str], Tuple[str, ...]] = () @@ -722,7 +726,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None): # Extra environment variables to pass to git commands self._environment: Dict[str, str] = {} - # cached command slots + # Cached command slots self.cat_file_header: Union[None, TBD] = None self.cat_file_all: Union[None, TBD] = None @@ -730,28 +734,30 @@ def __getattr__(self, name: str) -> Any: """A convenience method as it allows to call the command as if it was an object. - :return: Callable object that will execute call _call_process with your arguments.""" + :return: + Callable object that will execute call :meth:`_call_process` with + your arguments. + """ if name[0] == "_": return LazyMixin.__getattr__(self, name) return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) def set_persistent_git_options(self, **kwargs: Any) -> None: - """Specify command line options to the git executable - for subsequent subcommand calls. + """Specify command line options to the git executable for subsequent + subcommand calls. :param kwargs: - is a dict of keyword arguments. - These arguments are passed as in _call_process - but will be passed to the git command rather than - the subcommand. + A dict of keyword arguments. + These arguments are passed as in :meth:`_call_process`, but will be + passed to the git command rather than the subcommand. """ self._persistent_git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) def _set_cache_(self, attr: str) -> None: if attr == "_version_info": - # We only use the first 4 numbers, as everything else could be strings in fact (on windows) - process_version = self._call_process("version") # should be as default *args and **kwargs used + # We only use the first 4 numbers, as everything else could be strings in fact (on Windows). + process_version = self._call_process("version") # Should be as default *args and **kwargs used. version_numbers = process_version.split(" ")[2] self._version_info = cast( @@ -772,7 +778,9 @@ def version_info(self) -> Tuple[int, int, int, int]: """ :return: tuple(int, int, int, int) tuple with integers representing the major, minor and additional version numbers as parsed from git version. - This value is generated on demand and is cached.""" + + This value is generated on demand and is cached. + """ return self._version_info @overload @@ -839,7 +847,7 @@ def execute( strip_newline_in_stdout: bool = True, **subprocess_kwargs: Any, ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], AutoInterrupt]: - """Handles executing the command and consumes and returns the returned + R"""Handle executing the command, and consume and return the returned information (stdout). :param command: @@ -848,7 +856,7 @@ def execute( program to execute is the first item in the args sequence or string. :param istream: - Standard input filehandle passed to `subprocess.Popen`. + Standard input filehandle passed to :class:`subprocess.Popen`. :param with_extended_output: Whether to return a (status, stdout, stderr) tuple. @@ -858,17 +866,17 @@ def execute( :param as_process: Whether to return the created process instance directly from which - streams can be read on demand. This will render with_extended_output and - with_exceptions ineffective - the caller will have to deal with the details. - It is important to note that the process will be placed into an AutoInterrupt - wrapper that will interrupt the process once it goes out of scope. If you - use the command in iterators, you should pass the whole process instance - instead of a single stream. + streams can be read on demand. This will render `with_extended_output` + and `with_exceptions` ineffective - the caller will have to deal with + the details. It is important to note that the process will be placed + into an :class:`AutoInterrupt` wrapper that will interrupt the process + once it goes out of scope. If you use the command in iterators, you + should pass the whole process instance instead of a single stream. :param output_stream: If set to a file-like object, data produced by the git command will be output to the given stream directly. - This feature only has any effect if as_process is False. Processes will + This feature only has any effect if `as_process` is False. Processes will always be created with a pipe due to issues with subprocess. This merely is a workaround as data will be copied from the output pipe to the given output stream directly. @@ -881,13 +889,13 @@ def execute( :param kill_after_timeout: Specifies a timeout in seconds for the git command, after which the process - should be killed. This will have no effect if as_process is set to True. It is - set to None by default and will let the process run until the timeout is - explicitly specified. This feature is not supported on Windows. It's also worth - noting that kill_after_timeout uses SIGKILL, which can have negative side - effects on a repository. For example, stale locks in case of ``git gc`` could - render the repository incapable of accepting changes until the lock is manually - removed. + should be killed. This will have no effect if `as_process` is set to True. + It is set to None by default and will let the process run until the timeout + is explicitly specified. This feature is not supported on Windows. It's also + worth noting that `kill_after_timeout` uses SIGKILL, which can have negative + side effects on a repository. For example, stale locks in case of ``git gc`` + could render the repository incapable of accepting changes until the lock is + manually removed. :param with_stdout: If True, default True, we open stdout on the created process. @@ -901,7 +909,7 @@ def execute( It overrides :attr:`USE_SHELL` if it is not `None`. :param env: - A dictionary of environment variables to be passed to `subprocess.Popen`. + A dictionary of environment variables to be passed to :class:`subprocess.Popen`. :param max_chunk_size: Maximum number of bytes in one chunk of data passed to the output_stream in @@ -909,11 +917,11 @@ def execute( the default value is used. :param strip_newline_in_stdout: - Whether to strip the trailing ``\\n`` of the command stdout. + Whether to strip the trailing ``\n`` of the command stdout. :param subprocess_kwargs: - Keyword arguments to be passed to `subprocess.Popen`. Please note that - some of the valid kwargs are already set by this method; the ones you + Keyword arguments to be passed to :class:`subprocess.Popen`. Please note + that some of the valid kwargs are already set by this method; the ones you specify may not be the same ones. :return: @@ -931,8 +939,9 @@ def execute( :note: If you add additional keyword arguments to the signature of this method, - you must update the execute_kwargs tuple housed in this module.""" - # Remove password for the command if present + you must update the execute_kwargs tuple housed in this module. + """ + # Remove password for the command if present. redacted_command = remove_password_if_present(command) if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != "full" or as_process): log.info(" ".join(redacted_command)) @@ -945,12 +954,12 @@ def execute( except FileNotFoundError: cwd = None - # Start the process + # Start the process. inline_env = env env = os.environ.copy() - # Attempt to force all output to plain ascii english, which is what some parsing code - # may expect. - # According to stackoverflow (http://goo.gl/l74GC8), we are setting LANGUAGE as well + # Attempt to force all output to plain ASCII English, which is what some parsing + # code may expect. + # According to https://askubuntu.com/a/311796, we are setting LANGUAGE as well # just to be sure. env["LANGUAGE"] = "C" env["LC_ALL"] = "C" @@ -994,7 +1003,7 @@ def execute( stderr=PIPE, stdout=stdout_sink, shell=shell, - close_fds=is_posix, # unsupported on windows + close_fds=is_posix, # Unsupported on Windows. universal_newlines=universal_newlines, creationflags=PROC_CREATIONFLAGS, **subprocess_kwargs, @@ -1002,7 +1011,7 @@ def execute( except cmd_not_found_exception as err: raise GitCommandNotFound(redacted_command, err) from err else: - # replace with a typeguard for Popen[bytes]? + # Replace with a typeguard for Popen[bytes]? proc.stdout = cast(BinaryIO, proc.stdout) proc.stderr = cast(BinaryIO, proc.stderr) @@ -1024,7 +1033,7 @@ def kill_process(pid: int) -> None: if local_pid.isdigit(): child_pids.append(int(local_pid)) try: - # Windows does not have SIGKILL, so use SIGTERM instead + # Windows does not have SIGKILL, so use SIGTERM instead. sig = getattr(signal, "SIGKILL", signal.SIGTERM) os.kill(pid, sig) for child_pid in child_pids: @@ -1032,7 +1041,7 @@ def kill_process(pid: int) -> None: os.kill(child_pid, sig) except OSError: pass - kill_check.set() # tell the main routine that the process was killed + kill_check.set() # Tell the main routine that the process was killed. except OSError: # It is possible that the process gets completed in the duration after timeout # happens and before we try to kill the process. @@ -1045,7 +1054,7 @@ def kill_process(pid: int) -> None: kill_check = threading.Event() watchdog = threading.Timer(kill_after_timeout, kill_process, args=(proc.pid,)) - # Wait for the process to return + # Wait for the process to return. status = 0 stdout_value: Union[str, bytes] = b"" stderr_value: Union[str, bytes] = b"" @@ -1076,7 +1085,7 @@ def kill_process(pid: int) -> None: stream_copy(proc.stdout, output_stream, max_chunk_size) stdout_value = proc.stdout.read() stderr_value = proc.stderr.read() - # strip trailing "\n" + # Strip trailing "\n". if stderr_value.endswith(newline): # type: ignore stderr_value = stderr_value[:-1] status = proc.wait() @@ -1110,10 +1119,10 @@ def as_text(stdout_value: Union[bytes, str]) -> str: if with_exceptions and status != 0: raise GitCommandError(redacted_command, status, stderr_value, stdout_value) - if isinstance(stdout_value, bytes) and stdout_as_string: # could also be output_stream + if isinstance(stdout_value, bytes) and stdout_as_string: # Could also be output_stream. stdout_value = safe_decode(stdout_value) - # Allow access to the command's status code + # Allow access to the command's status code. if with_extended_output: return (status, stdout_value, safe_decode(stderr_value)) else: @@ -1123,18 +1132,18 @@ def environment(self) -> Dict[str, str]: return self._environment def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: - """ - Set environment variables for future git invocations. Return all changed - values in a format that can be passed back into this function to revert - the changes: + """Set environment variables for future git invocations. Return all changed + values in a format that can be passed back into this function to revert the + changes. ``Examples``:: old_env = self.update_environment(PWD='/tmp') self.update_environment(**old_env) - :param kwargs: environment variables to use for git processes - :return: dict that maps environment variables to their old values + :param kwargs: Environment variables to use for git processes + + :return: Dict that maps environment variables to their old values """ old_env = {} for key, value in kwargs.items(): @@ -1150,16 +1159,15 @@ def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: @contextlib.contextmanager def custom_environment(self, **kwargs: Any) -> Iterator[None]: - """ - A context manager around the above ``update_environment`` method to restore the - environment back to its previous state after operation. + """A context manager around the above :meth:`update_environment` method to + restore the environment back to its previous state after operation. ``Examples``:: with self.custom_environment(GIT_SSH='/bin/ssh_wrapper'): repo.remotes.origin.fetch() - :param kwargs: see update_environment + :param kwargs: See :meth:`update_environment` """ old_env = self.update_environment(**kwargs) try: @@ -1184,7 +1192,7 @@ def transform_kwarg(self, name: str, value: Any, split_single_char_options: bool return [] def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any) -> List[str]: - """Transforms Python style kwargs into git command line options.""" + """Transform Python style kwargs into git command line options.""" args = [] for k, v in kwargs.items(): if isinstance(v, (list, tuple)): @@ -1206,23 +1214,22 @@ def _unpack_args(cls, arg_list: Sequence[str]) -> List[str]: return outlist def __call__(self, **kwargs: Any) -> "Git": - """Specify command line options to the git executable - for a subcommand call. + """Specify command line options to the git executable for a subcommand call. :param kwargs: - is a dict of keyword arguments. - these arguments are passed as in _call_process - but will be passed to the git command rather than - the subcommand. + A dict of keyword arguments. + These arguments are passed as in :meth:`_call_process`, but will be + passed to the git command rather than the subcommand. ``Examples``:: - git(work_tree='/tmp').difftool()""" + git(work_tree='/tmp').difftool() + """ self._git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) return self @overload def _call_process(self, method: str, *args: None, **kwargs: None) -> str: - ... # if no args given, execute called with all defaults + ... # If no args were given, execute the call with all defaults. @overload def _call_process( @@ -1248,20 +1255,20 @@ def _call_process( the result as a string. :param method: - is the command. Contained "_" characters will be converted to dashes, - such as in 'ls_files' to call 'ls-files'. + The command. Contained ``_`` characters will be converted to dashes, + such as in ``ls_files`` to call ``ls-files``. :param args: - is the list of arguments. If None is included, it will be pruned. + The list of arguments. If None is included, it will be pruned. This allows your commands to call git more conveniently as None is realized as non-existent. :param kwargs: - It contains key-values for the following: - - the :meth:`execute()` kwds, as listed in :var:`execute_kwargs`; - - "command options" to be converted by :meth:`transform_kwargs()`; - - the `'insert_kwargs_after'` key which its value must match one of ``*args`` - and any cmd-options will be appended after the matched arg. + Contains key-values for the following: + - The :meth:`execute()` kwds, as listed in :var:`execute_kwargs`. + - "Command options" to be converted by :meth:`transform_kwargs`. + - The `'insert_kwargs_after'` key which its value must match one of ``*args``. + It also contains any command options, to be appended after the matched arg. Examples:: @@ -1271,17 +1278,18 @@ def _call_process( git rev-list max-count 10 --header master - :return: Same as ``execute`` - if no args given used execute default (esp. as_process = False, stdout_as_string = True) - and return str""" - # Handle optional arguments prior to calling transform_kwargs - # otherwise these'll end up in args, which is bad. + :return: Same as :meth:`execute`. + If no args are given, used :meth:`execute`'s default (especially + ``as_process = False``, ``stdout_as_string = True``) and return str. + """ + # Handle optional arguments prior to calling transform_kwargs. + # Otherwise these'll end up in args, which is bad. exec_kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} opts_kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} insert_after_this_arg = opts_kwargs.pop("insert_kwargs_after", None) - # Prepare the argument list + # Prepare the argument list. opt_args = self.transform_kwargs(**opts_kwargs) ext_args = self._unpack_args([a for a in args if a is not None]) @@ -1302,11 +1310,10 @@ def _call_process( call = [self.GIT_PYTHON_GIT_EXECUTABLE] - # add persistent git options + # Add persistent git options. call.extend(self._persistent_git_options) - # add the git options, then reset to empty - # to avoid side_effects + # Add the git options, then reset to empty to avoid side effects. call.extend(self._git_options) self._git_options = () @@ -1322,7 +1329,7 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: :return: (hex_sha, type_string, size_as_int) - :raise ValueError: if the header contains indication for an error due to + :raise ValueError: If the header contains indication for an error due to incorrect input sha""" tokens = header_line.split() if len(tokens) != 3: @@ -1338,12 +1345,12 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: return (tokens[0], tokens[1], int(tokens[2])) def _prepare_ref(self, ref: AnyStr) -> bytes: - # required for command to separate refs on stdin, as bytes + # Required for command to separate refs on stdin, as bytes. if isinstance(ref, bytes): - # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text + # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text. refstr: str = ref.decode("ascii") elif not isinstance(ref, str): - refstr = str(ref) # could be ref-object + refstr = str(ref) # Could be ref-object. else: refstr = ref @@ -1379,7 +1386,8 @@ def get_object_header(self, ref: str) -> Tuple[str, str, int]: :note: The method will only suffer from the costs of command invocation once and reuses the command in subsequent calls. - :return: (hexsha, type_string, size_as_int)""" + :return: (hexsha, type_string, size_as_int) + """ cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) @@ -1387,7 +1395,8 @@ def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: """As get_object_header, but returns object data as well. :return: (hexsha, type_string, size_as_int, data_string) - :note: not threadsafe""" + :note: Not threadsafe. + """ hexsha, typename, size, stream = self.stream_object_data(ref) data = stream.read(size) del stream @@ -1397,7 +1406,8 @@ def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileConte """As get_object_header, but returns the data as a stream. :return: (hexsha, type_string, size_as_int, stream) - :note: This method is not threadsafe, you need one independent Command instance per thread to be safe!""" + :note: This method is not threadsafe, you need one independent Command instance per thread to be safe! + """ cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) cmd_stdout = cmd.stdout if cmd.stdout is not None else io.BytesIO() @@ -1408,7 +1418,8 @@ def clear_cache(self) -> "Git": Currently persistent commands will be interrupted. - :return: self""" + :return: self + """ for cmd in (self.cat_file_all, self.cat_file_header): if cmd: cmd.__del__() diff --git a/git/compat.py b/git/compat.py index 624f26116..93e06d2ec 100644 --- a/git/compat.py +++ b/git/compat.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- -# config.py +# compat.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ -"""utilities to help provide compatibility with python 3""" + +"""Utilities to help provide compatibility with Python 3.""" + # flake8: noqa import locale @@ -50,7 +52,7 @@ def safe_decode(s: AnyStr) -> str: def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: - """Safely decodes a binary string to unicode""" + """Safely decode a binary string to Unicode.""" if isinstance(s, str): return s elif isinstance(s, bytes): @@ -72,7 +74,7 @@ def safe_encode(s: AnyStr) -> bytes: def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: - """Safely encodes a binary string to unicode""" + """Safely encode a binary string to Unicode.""" if isinstance(s, str): return s.encode(defenc) elif isinstance(s, bytes): @@ -94,7 +96,7 @@ def win_encode(s: AnyStr) -> bytes: def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: - """Encode unicodes for process arguments on Windows.""" + """Encode Unicode strings for process arguments on Windows.""" if isinstance(s, str): return s.encode(locale.getpreferredencoding(False)) elif isinstance(s, bytes): diff --git a/git/config.py b/git/config.py index 76b149179..31b8a665d 100644 --- a/git/config.py +++ b/git/config.py @@ -3,8 +3,9 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + """Module containing module parser implementation able to properly read and write -configuration files""" +configuration files.""" import sys import abc @@ -85,12 +86,12 @@ class MetaParserBuilder(abc.ABCMeta): # noqa: B024 - """Utility class wrapping base-class methods into decorators that assure read-only properties""" + """Utility class wrapping base-class methods into decorators that assure read-only properties.""" def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder": + """Equip all base-class methods with a needs_values decorator, and all non-const + methods with a set_dirty_and_flush_changes decorator in addition to that. """ - Equip all base-class methods with a needs_values decorator, and all non-const methods - with a set_dirty_and_flush_changes decorator in addition to that.""" kmm = "_mutating_methods_" if kmm in clsdict: mutating_methods = clsdict[kmm] @@ -114,7 +115,7 @@ def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParse def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: - """Returns method assuring we read values (on demand) before we try to access them""" + """Return a method for ensuring we read values (on demand) before we try to access them.""" @wraps(func) def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: @@ -126,9 +127,10 @@ def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _ def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: - """Return method that checks whether given non constant function may be called. - If so, the instance will be set dirty. - Additionally, we flush the changes right to disk""" + """Return a method that checks whether given non constant function may be called. + + If so, the instance will be set dirty. Additionally, we flush the changes right to disk. + """ def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: rval = non_const_func(self, *args, **kwargs) @@ -142,14 +144,13 @@ def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: class SectionConstraint(Generic[T_ConfigParser]): - """Constrains a ConfigParser to only option commands which are constrained to always use the section we have been initialized with. It supports all ConfigParser methods that operate on an option. - :note: - If used as a context manager, will release the wrapped ConfigParser.""" + :note: If used as a context manager, will release the wrapped ConfigParser. + """ __slots__ = ("_config", "_section_name") _valid_attrs_ = ( @@ -183,16 +184,17 @@ def __getattr__(self, attr: str) -> Any: def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any: """Call the configuration at the given method which must take a section name - as first argument""" + as first argument.""" return getattr(self._config, method)(self._section_name, *args, **kwargs) @property def config(self) -> T_ConfigParser: - """return: Configparser instance we constrain""" + """return: ConfigParser instance we constrain""" return self._config def release(self) -> None: - """Equivalent to GitConfigParser.release(), which is called on our underlying parser instance""" + """Equivalent to GitConfigParser.release(), which is called on our underlying + parser instance.""" return self._config.release() def __enter__(self) -> "SectionConstraint[T_ConfigParser]": @@ -248,8 +250,8 @@ def items_all(self) -> List[Tuple[str, List[_T]]]: def get_config_path(config_level: Lit_config_levels) -> str: - # we do not support an absolute path of the gitconfig on windows , - # use the global config instead + # We do not support an absolute path of the gitconfig on Windows. + # Use the global config instead. if is_win and config_level == "system": config_level = "global" @@ -271,7 +273,6 @@ def get_config_path(config_level: Lit_config_levels) -> str: class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): - """Implements specifics required to read git style configuration files. This variation behaves much like the git.config command such that the configuration @@ -286,7 +287,10 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): :note: The config is case-sensitive even when queried, hence section and option names must match perfectly. - If used as a context manager, will release the locked file.""" + + :note: + If used as a context manager, this will release the locked file. + """ # { Configuration # The lock type determines the type of lock to use in new configuration readers. @@ -317,29 +321,34 @@ def __init__( repo: Union["Repo", None] = None, ) -> None: """Initialize a configuration reader to read the given file_or_files and to - possibly allow changes to it by setting read_only False + possibly allow changes to it by setting read_only False. :param file_or_files: - A single file path or file objects or multiple of these + A file path or file object, or a sequence of possibly more than one of them. :param read_only: - If True, the ConfigParser may only read the data , but not change it. - If False, only a single file path or file object may be given. We will write back the changes - when they happen, or when the ConfigParser is released. This will not happen if other - configuration files have been included - :param merge_includes: if True, we will read files mentioned in [include] sections and merge their - contents into ours. This makes it impossible to write back an individual configuration file. - Thus, if you want to modify a single configuration file, turn this off to leave the original - dataset unaltered when reading it. - :param repo: Reference to repository to use if [includeIf] sections are found in configuration files. - + If True, the ConfigParser may only read the data, but not change it. + If False, only a single file path or file object may be given. We will write + back the changes when they happen, or when the ConfigParser is released. + This will not happen if other configuration files have been included. + + :param merge_includes: + If True, we will read files mentioned in ``[include]`` sections and merge + their contents into ours. This makes it impossible to write back an + individual configuration file. Thus, if you want to modify a single + configuration file, turn this off to leave the original dataset unaltered + when reading it. + + :param repo: + Reference to repository to use if ``[includeIf]`` sections are found in + configuration files. """ cp.RawConfigParser.__init__(self, dict_type=_OMD) self._dict: Callable[..., _OMD] # type: ignore # mypy/typeshed bug? self._defaults: _OMD self._sections: _OMD # type: ignore # mypy/typeshed bug? - # Used in python 3, needs to stay in sync with sections for underlying implementation to work + # Used in Python 3. Needs to stay in sync with sections for underlying implementation to work. if not hasattr(self, "_proxies"): self._proxies = self._dict() @@ -377,7 +386,7 @@ def _acquire_lock(self) -> None: file_or_files = self._file_or_files.name # END get filename from handle/stream - # initialize lock base - we want to write + # Initialize lock base - we want to write. self._lock = self.t_lock(file_or_files) # END lock check @@ -386,7 +395,7 @@ def _acquire_lock(self) -> None: def __del__(self) -> None: """Write pending changes if required and release locks""" - # NOTE: only consistent in PY2 + # NOTE: Only consistent in Python 2. self.release() def __enter__(self) -> "GitConfigParser": @@ -398,10 +407,11 @@ def __exit__(self, *args: Any) -> None: def release(self) -> None: """Flush changes and release the configuration write lock. This instance must not be used anymore afterwards. + In Python 3, it's required to explicitly release locks and flush changes, as __del__ is not called deterministically anymore.""" - # checking for the lock here makes sure we do not raise during write() - # in case an invalid parser was created who could not get a lock + # Checking for the lock here makes sure we do not raise during write() + # in case an invalid parser was created who could not get a lock. if self.read_only or (self._lock and not self._lock._has_lock()): return @@ -410,33 +420,31 @@ def release(self) -> None: except IOError: log.error("Exception during destruction of GitConfigParser", exc_info=True) except ReferenceError: - # This happens in PY3 ... and usually means that some state cannot be - # written as the sections dict cannot be iterated - # Usually when shutting down the interpreter, don't know how to fix this + # This happens in Python 3... and usually means that some state cannot be + # written as the sections dict cannot be iterated. This usually happens when + # the interpreter is shutting down. Can it be fixed? pass finally: if self._lock is not None: self._lock._release_lock() def optionxform(self, optionstr: str) -> str: - """Do not transform options in any way when writing""" + """Do not transform options in any way when writing.""" return optionstr def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: - """A direct copy of the py2.4 version of the super class's _read method - to assure it uses ordered dicts. Had to change one line to make it work. - - Future versions have this fixed, but in fact its quite embarrassing for the - guys not to have done it right in the first place ! - - Removed big comments to make it more compact. + """Originally a direct copy of the Python 2.4 version of RawConfigParser._read, + to ensure it uses ordered dicts. - Made sure it ignores initial whitespace as git uses tabs""" - cursect = None # None, or a dictionary + The ordering bug was fixed in Python 2.4, and dict itself keeps ordering since + Python 3.7. This has some other changes, especially that it ignores initial + whitespace, since git uses tabs. (Big comments are removed to be more compact.) + """ + cursect = None # None, or a dictionary. optname = None lineno = 0 is_multi_line = False - e = None # None, or an exception + e = None # None, or an exception. def string_decode(v: str) -> str: if v[-1] == "\\": @@ -449,19 +457,19 @@ def string_decode(v: str) -> str: # end while True: - # we assume to read binary ! + # We assume to read binary! line = fp.readline().decode(defenc) if not line: break lineno = lineno + 1 - # comment or blank line? + # Comment or blank line? if line.strip() == "" or self.re_comment.match(line): continue if line.split(None, 1)[0].lower() == "rem" and line[0] in "rR": - # no leading whitespace + # No leading whitespace. continue - # is it a section header? + # Is it a section header? mo = self.SECTCRE.match(line.strip()) if not is_multi_line and mo: sectname: str = mo.group("header").strip() @@ -473,16 +481,16 @@ def string_decode(v: str) -> str: cursect = self._dict((("__name__", sectname),)) self._sections[sectname] = cursect self._proxies[sectname] = None - # So sections can't start with a continuation line + # So sections can't start with a continuation line. optname = None - # no section header in the file? + # No section header in the file? elif cursect is None: raise cp.MissingSectionHeaderError(fpname, lineno, line) - # an option line? + # An option line? elif not is_multi_line: mo = self.OPTCRE.match(line) if mo: - # We might just have handled the last line, which could contain a quotation we want to remove + # We might just have handled the last line, which could contain a quotation we want to remove. optname, vi, optval = mo.group("option", "vi", "value") if vi in ("=", ":") and ";" in optval and not optval.strip().startswith('"'): pos = optval.find(";") @@ -497,10 +505,10 @@ def string_decode(v: str) -> str: is_multi_line = True optval = string_decode(optval[1:]) # end handle multi-line - # preserves multiple values for duplicate optnames + # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) else: - # check if it's an option with no value - it's just ignored by git + # Check if it's an option with no value - it's just ignored by git. if not self.OPTVALUEONLY.match(line): if not e: e = cp.ParsingError(fpname) @@ -517,7 +525,7 @@ def string_decode(v: str) -> str: # END parse section or option # END while reading - # if any parsing errors occurred, raise an exception + # If any parsing errors occurred, raise an exception. if e: raise e @@ -525,8 +533,9 @@ def _has_includes(self) -> Union[bool, int]: return self._merge_includes and len(self._included_paths()) def _included_paths(self) -> List[Tuple[str, str]]: - """Return List all paths that must be included to configuration - as Tuples of (option, value). + """List all paths that must be included to configuration. + + :return: The list of paths, where each path is a tuple of ``(option, value)``. """ paths = [] @@ -573,21 +582,24 @@ def _included_paths(self) -> List[Tuple[str, str]]: return paths def read(self) -> None: # type: ignore[override] - """Reads the data stored in the files we have been initialized with. It will - ignore files that cannot be read, possibly leaving an empty configuration + """Read the data stored in the files we have been initialized with. + + This will ignore files that cannot be read, possibly leaving an empty + configuration. :return: Nothing - :raise IOError: if a file cannot be handled""" + :raise IOError: If a file cannot be handled + """ if self._is_initialized: return None self._is_initialized = True files_to_read: List[Union[PathLike, IO]] = [""] if isinstance(self._file_or_files, (str, os.PathLike)): - # for str or Path, as str is a type of Sequence + # For str or Path, as str is a type of Sequence. files_to_read = [self._file_or_files] elif not isinstance(self._file_or_files, (tuple, list, Sequence)): - # could merge with above isinstance once runtime type known + # Could merge with above isinstance once runtime type known. files_to_read = [self._file_or_files] else: # for lists or tuples files_to_read = list(self._file_or_files) @@ -600,11 +612,11 @@ def read(self) -> None: # type: ignore[override] file_ok = False if hasattr(file_path, "seek"): - # must be a file objectfile-object - file_path = cast(IO[bytes], file_path) # replace with assert to narrow type, once sure + # Must be a file objectfile-object. + file_path = cast(IO[bytes], file_path) # TODO: Replace with assert to narrow type, once sure. self._read(file_path, file_path.name) else: - # assume a path if it is not a file-object + # Assume a path if it is not a file-object. file_path = cast(PathLike, file_path) try: with open(file_path, "rb") as fp: @@ -613,8 +625,8 @@ def read(self) -> None: # type: ignore[override] except IOError: continue - # Read includes and append those that we didn't handle yet - # We expect all paths to be normalized and absolute (and will assure that is the case) + # Read includes and append those that we didn't handle yet. + # We expect all paths to be normalized and absolute (and will assure that is the case). if self._has_includes(): for _, include_path in self._included_paths(): if include_path.startswith("~"): @@ -631,27 +643,27 @@ def read(self) -> None: # type: ignore[override] if include_path in seen or not os.access(include_path, os.R_OK): continue seen.add(include_path) - # insert included file to the top to be considered first + # Insert included file to the top to be considered first. files_to_read.insert(0, include_path) num_read_include_files += 1 # each include path in configuration file # end handle includes # END for each file object to read - # If there was no file included, we can safely write back (potentially) the configuration file - # without altering it's meaning + # If there was no file included, we can safely write back (potentially) the + # configuration file without altering its meaning. if num_read_include_files == 0: self._merge_includes = False # end def _write(self, fp: IO) -> None: """Write an .ini-format representation of the configuration state in - git compatible format""" + git compatible format.""" def write_section(name: str, section_dict: _OMD) -> None: fp.write(("[%s]\n" % name).encode(defenc)) - values: Sequence[str] # runtime only gets str in tests, but should be whatever _OMD stores + values: Sequence[str] # Runtime only gets str in tests, but should be whatever _OMD stores. v: str for key, values in section_dict.items_all(): if key == "__name__": @@ -692,9 +704,9 @@ def items_all(self, section_name: str) -> List[Tuple[str, List[str]]]: @needs_values def write(self) -> None: - """Write changes to our file, if there are changes at all + """Write changes to our file, if there are changes at all. - :raise IOError: if this is a read-only writer instance or if we could not obtain + :raise IOError: If this is a read-only writer instance or if we could not obtain a file lock""" self._assure_writable("write") if not self._dirty: @@ -717,9 +729,9 @@ def write(self) -> None: fp = self._file_or_files - # we have a physical file on disk, so get a lock - is_file_lock = isinstance(fp, (str, os.PathLike, IOBase)) # can't use Pathlike until 3.5 dropped - if is_file_lock and self._lock is not None: # else raise Error? + # We have a physical file on disk, so get a lock. + is_file_lock = isinstance(fp, (str, os.PathLike, IOBase)) # TODO: Use PathLike (having dropped 3.5). + if is_file_lock and self._lock is not None: # Else raise error? self._lock._obtain_lock() if not hasattr(fp, "seek"): @@ -729,7 +741,7 @@ def write(self) -> None: else: fp = cast("BytesIO", fp) fp.seek(0) - # make sure we do not overwrite into an existing file + # Make sure we do not overwrite into an existing file. if hasattr(fp, "truncate"): fp.truncate() self._write(fp) @@ -747,13 +759,13 @@ def read_only(self) -> bool: """:return: True if this instance may change the configuration file""" return self._read_only + # FIXME: Figure out if default or return type can really include bool. def get_value( self, section: str, option: str, default: Union[int, float, str, bool, None] = None, ) -> Union[int, float, str, bool]: - # can default or return type include bool? """Get an option's value. If multiple values are specified for this option in the section, the @@ -762,10 +774,12 @@ def get_value( :param default: If not None, the given default value will be returned in case the option did not exist + :return: a properly typed value, either int, float or string :raise TypeError: in case the value could not be understood - Otherwise the exceptions known to the ConfigParser will be raised.""" + Otherwise the exceptions known to the ConfigParser will be raised. + """ try: valuestr = self.get(section, option) except Exception: @@ -789,10 +803,12 @@ def get_values( :param default: If not None, a list containing the given default value will be returned in case the option did not exist + :return: a list of properly typed values, either int, float or string :raise TypeError: in case the value could not be understood - Otherwise the exceptions known to the ConfigParser will be raised.""" + Otherwise the exceptions known to the ConfigParser will be raised. + """ try: self.sections() lst = self._sections[section].getall(option) @@ -816,7 +832,7 @@ def _string_to_value(self, valuestr: str) -> Union[int, float, str, bool]: continue # END for each numeric type - # try boolean values as git uses them + # Try boolean values as git uses them. vl = valuestr.lower() if vl == "false": return False @@ -839,16 +855,17 @@ def _value_to_string(self, value: Union[str, bytes, int, float, bool]) -> str: @needs_values @set_dirty_and_flush_changes def set_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser": - """Sets the given option in section to the given value. - It will create the section if required, and will not throw as opposed to the default - ConfigParser 'set' method. + """Set the given option in section to the given value. + + This will create the section if required, and will not throw as opposed to the + default ConfigParser 'set' method. :param section: Name of the section in which the option resides or should reside :param option: Name of the options whose value to set - - :param value: Value to set the option to. It must be a string or convertible - to a string - :return: this instance""" + :param value: Value to set the option to. It must be a string or convertible to + a string. + :return: This instance + """ if not self.has_section(section): self.add_section(section) self.set(section, option, self._value_to_string(value)) @@ -857,27 +874,29 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo @needs_values @set_dirty_and_flush_changes def add_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser": - """Adds a value for the given option in section. - It will create the section if required, and will not throw as opposed to the default + """Add a value for the given option in section. + + This will create the section if required, and will not throw as opposed to the default ConfigParser 'set' method. The value becomes the new value of the option as returned by 'get_value', and appends to the list of values returned by 'get_values`'. :param section: Name of the section in which the option resides or should reside :param option: Name of the option - :param value: Value to add to option. It must be a string or convertible to a string - :return: this instance""" + :return: This instance + """ if not self.has_section(section): self.add_section(section) self._sections[section].add(option, self._value_to_string(value)) return self def rename_section(self, section: str, new_name: str) -> "GitConfigParser": - """rename the given section to new_name - :raise ValueError: if section doesn't exit - :raise ValueError: if a section with new_name does already exist - :return: this instance + """Rename the given section to new_name. + + :raise ValueError: If ``section`` doesn't exist + :raise ValueError: If a section with ``new_name`` does already exist + :return: This instance """ if not self.has_section(section): raise ValueError("Source section '%s' doesn't exist" % section) @@ -890,6 +909,6 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": new_section.setall(k, vs) # end for each value to copy - # This call writes back the changes, which is why we don't have the respective decorator + # This call writes back the changes, which is why we don't have the respective decorator. self.remove_section(section) return self diff --git a/git/db.py b/git/db.py index b1a0d108a..1aacd0c84 100644 --- a/git/db.py +++ b/git/db.py @@ -1,4 +1,5 @@ -"""Module with our own gitdb implementation - it uses the git command""" +"""Module with our own gitdb implementation - it uses the git command.""" + from git.util import bin_to_hex, hex_to_bin from gitdb.base import OInfo, OStream from gitdb.db import GitDB @@ -22,17 +23,17 @@ class GitCmdObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file + objects, pack files and an alternates file. It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations + + :note: For now, we use the git command to do all the lookup, just until we + have packs and the other implementations. """ def __init__(self, root_path: PathLike, git: "Git") -> None: - """Initialize this instance with the root and a git command""" + """Initialize this instance with the root and a git command.""" super(GitCmdObjectDB, self).__init__(root_path) self._git = git @@ -48,11 +49,15 @@ def stream(self, binsha: bytes) -> OStream: # { Interface def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: - """:return: Full binary 20 byte sha from the given partial hexsha + """ + :return: Full binary 20 byte sha from the given partial hexsha + :raise AmbiguousObjectName: :raise BadObject: - :note: currently we only raise BadObject as git does not communicate - AmbiguousObjects separately""" + + :note: Currently we only raise :class:`BadObject` as git does not communicate + AmbiguousObjects separately. + """ try: hexsha, _typename, _size = self._git.get_object_header(partial_hexsha) return hex_to_bin(hexsha) diff --git a/git/diff.py b/git/diff.py index 8ee2527d4..1fde4d676 100644 --- a/git/diff.py +++ b/git/diff.py @@ -50,7 +50,7 @@ __all__ = ("Diffable", "DiffIndex", "Diff", "NULL_TREE") -# Special object to compare against the empty tree in diffs +# Special object to compare against the empty tree in diffs. NULL_TREE = object() _octal_byte_re = re.compile(rb"\\([0-9]{3})") @@ -80,27 +80,28 @@ def decode_path(path: bytes, has_ab_prefix: bool = True) -> Optional[bytes]: class Diffable(object): - - """Common interface for all object that can be diffed against another object of compatible type. + """Common interface for all objects that can be diffed against another object of + compatible type. :note: - Subclasses require a repo member as it is the case for Object instances, for practical - reasons we do not derive from Object.""" + Subclasses require a repo member as it is the case for Object instances, for + practical reasons we do not derive from Object. + """ __slots__ = () - # standin indicating you want to diff against the index class Index(object): - pass + """Stand-in indicating you want to diff against the index.""" def _process_diff_args( self, args: List[Union[str, "Diffable", Type["Diffable.Index"], object]] ) -> List[Union[str, "Diffable", Type["Diffable.Index"], object]]: """ :return: - possibly altered version of the given args list. - Method is called right before git command execution. - Subclasses can use it to alter the behaviour of the superclass""" + Possibly altered version of the given args list. + This method is called right before git command execution. + Subclasses can use it to alter the behaviour of the superclass. + """ return args def diff( @@ -110,41 +111,47 @@ def diff( create_patch: bool = False, **kwargs: Any, ) -> "DiffIndex": - """Creates diffs between two items being trees, trees and index or an - index and the working tree. It will detect renames automatically. + """Create diffs between two items being trees, trees and index or an + index and the working tree. Detects renames automatically. :param other: - Is the item to compare us with. - If None, we will be compared to the working tree. - If Treeish, it will be compared against the respective tree - If Index ( type ), it will be compared against the index. - If git.NULL_TREE, it will compare against the empty tree. - It defaults to Index to assure the method will not by-default fail - on bare repositories. + This the item to compare us with. + + * If None, we will be compared to the working tree. + * If :class:`Treeish `, it will be compared against + the respective tree. + * If :class:`Index `, it will be compared against the index. + * If :attr:`git.NULL_TREE`, it will compare against the empty tree. + * It defaults to :class:`Index ` to assure the method will + not by-default fail on bare repositories. :param paths: - is a list of paths or a single path to limit the diff to. - It will only include at least one of the given path or paths. + This a list of paths or a single path to limit the diff to. It will only + include at least one of the given path or paths. :param create_patch: - If True, the returned Diff contains a detailed patch that if applied - makes the self to other. Patches are somewhat costly as blobs have to be read - and diffed. + If True, the returned :class:`Diff` contains a detailed patch that if + applied makes the self to other. Patches are somewhat costly as blobs have + to be read and diffed. :param kwargs: - Additional arguments passed to git-diff, such as - R=True to swap both sides of the diff. + Additional arguments passed to git-diff, such as ``R=True`` to swap both + sides of the diff. :return: git.DiffIndex :note: - On a bare repository, 'other' needs to be provided as Index or as - as Tree/Commit, or a git command error will occur""" + On a bare repository, 'other' needs to be provided + as :class:`Index `, + or as :class:`Tree ` + or :class:`Commit `, or a git command error will + occur. + """ args: List[Union[PathLike, Diffable, Type["Diffable.Index"], object]] = [] - args.append("--abbrev=40") # we need full shas - args.append("--full-index") # get full index paths, not only filenames + args.append("--abbrev=40") # We need full shas. + args.append("--full-index") # Get full index paths, not only filenames. - # remove default '-M' arg (check for renames) if user is overriding it + # Remove default '-M' arg (check for renames) if user is overriding it. if not any(x in kwargs for x in ("find_renames", "no_renames", "M")): args.append("-M") @@ -154,8 +161,8 @@ def diff( args.append("--raw") args.append("-z") - # in any way, assure we don't see colored output, - # fixes https://github.com/gitpython-developers/GitPython/issues/172 + # Ensure we never see colored output. + # Fixes: https://github.com/gitpython-developers/GitPython/issues/172 args.append("--no-color") if paths is not None and not isinstance(paths, (tuple, list)): @@ -168,17 +175,17 @@ def diff( if other is self.Index: args.insert(0, "--cached") elif other is NULL_TREE: - args.insert(0, "-r") # recursive diff-tree + args.insert(0, "-r") # Recursive diff-tree. args.insert(0, "--root") diff_cmd = self.repo.git.diff_tree elif other is not None: - args.insert(0, "-r") # recursive diff-tree + args.insert(0, "-r") # Recursive diff-tree. args.insert(0, other) diff_cmd = self.repo.git.diff_tree args.insert(0, self) - # paths is list here or None + # paths is list here, or None. if paths: args.append("--") args.extend(paths) @@ -198,13 +205,13 @@ def diff( class DiffIndex(List[T_Diff]): + """An Index for diffs, allowing a list of Diffs to be queried by the diff + properties. - """Implements an Index for diffs, allowing a list of Diffs to be queried by - the diff properties. - - The class improves the diff handling convenience""" + The class improves the diff handling convenience. + """ - # change type invariant identifying possible ways a blob can have changed + # Change type invariant identifying possible ways a blob can have changed: # A = Added # D = Deleted # R = Renamed @@ -215,10 +222,10 @@ class DiffIndex(List[T_Diff]): def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: """ :return: - iterator yielding Diff instances that match the given change_type + iterator yielding :class:`Diff` instances that match the given `change_type` :param change_type: - Member of DiffIndex.change_type, namely: + Member of :attr:`DiffIndex.change_type`, namely: * 'A' for added paths * 'D' for deleted paths @@ -246,11 +253,10 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: class Diff(object): - """A Diff contains diff information between two Trees. It contains two sides a and b of the diff, members are prefixed with - "a" and "b" respectively to inidcate that. + "a" and "b" respectively to indicate that. Diffs keep information about the changed blob objects, the file mode, renames, deletions and new files. @@ -273,11 +279,13 @@ class Diff(object): When comparing to working trees, the working tree blob will have a null hexsha as a corresponding object does not yet exist. The mode will be null as well. - But the path will be available though. - If it is listed in a diff the working tree version of the file must - be different to the version in the index or tree, and hence has been modified.""" + The path will be available, though. + + If it is listed in a diff, the working tree version of the file must + differ from the version in the index or tree, and hence has been modified. + """ - # precompiled regex + # Precompiled regex. re_header = re.compile( rb""" ^diff[ ]--git @@ -299,7 +307,8 @@ class Diff(object): """, re.VERBOSE | re.MULTILINE, ) - # can be used for comparisons + + # These can be used for comparisons. NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = b"\0" * 20 @@ -346,8 +355,8 @@ def __init__( self.a_mode = mode_str_to_int(a_mode) if a_mode else None self.b_mode = mode_str_to_int(b_mode) if b_mode else None - # Determine whether this diff references a submodule, if it does then - # we need to overwrite "repo" to the corresponding submodule's repo instead + # Determine whether this diff references a submodule. If it does then + # we need to overwrite "repo" to the corresponding submodule's repo instead. if repo and a_rawpath: for submodule in repo.submodules: if submodule.path == a_rawpath.decode(defenc, "replace"): @@ -371,7 +380,7 @@ def __init__( self.deleted_file: bool = deleted_file self.copied_file: bool = copied_file - # be clear and use None instead of empty strings + # Be clear and use None instead of empty strings. assert raw_rename_from is None or isinstance(raw_rename_from, bytes) assert raw_rename_to is None or isinstance(raw_rename_to, bytes) self.raw_rename_from = raw_rename_from or None @@ -395,15 +404,15 @@ def __hash__(self) -> int: return hash(tuple(getattr(self, n) for n in self.__slots__)) def __str__(self) -> str: - h: str = "%s" + h = "%s" if self.a_blob: h %= self.a_blob.path elif self.b_blob: h %= self.b_blob.path - msg: str = "" - line = None # temp line - line_length = 0 # line length + msg = "" + line = None + line_length = 0 for b, n in zip((self.a_blob, self.b_blob), ("lhs", "rhs")): if b: line = "\n%s: %o | %s" % (n, b.mode, b.hexsha) @@ -414,7 +423,7 @@ def __str__(self) -> str: msg += line # END for each blob - # add headline + # Add headline. h += "\n" + "=" * line_length if self.deleted_file: @@ -437,11 +446,7 @@ def __str__(self) -> str: msg += "\n---" # END diff info - # Python2 silliness: have to assure we convert our likely to be unicode object to a string with the - # right encoding. Otherwise it tries to convert it using ascii, which may fail ungracefully - res = h + msg - # end - return res + return h + msg @property def a_path(self) -> Optional[str]: @@ -461,8 +466,11 @@ def rename_to(self) -> Optional[str]: @property def renamed(self) -> bool: - """:returns: True if the blob of our diff has been renamed - :note: This property is deprecated, please use ``renamed_file`` instead. + """ + :returns: True if the blob of our diff has been renamed + + :note: This property is deprecated. + Please use the :attr:`renamed_file` property instead. """ return self.renamed_file @@ -493,17 +501,17 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn :return: git.DiffIndex """ - ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. + # FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text_list: List[bytes] = [] handle_process_output(proc, text_list.append, None, finalize_process, decode_streams=False) - # for now, we have to bake the stream + # For now, we have to bake the stream. text = b"".join(text_list) index: "DiffIndex" = DiffIndex() previous_header: Union[Match[bytes], None] = None header: Union[Match[bytes], None] = None - a_path, b_path = None, None # for mypy - a_mode, b_mode = None, None # for mypy + a_path, b_path = None, None # For mypy. + a_mode, b_mode = None, None # For mypy. for _header in cls.re_header.finditer(text): ( a_path_fallback, @@ -532,13 +540,13 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback) # Our only means to find the actual text is to see what has not been matched by our regex, - # and then retro-actively assign it to our index + # and then retro-actively assign it to our index. if previous_header is not None: index[-1].diff = text[previous_header.end() : _header.start()] # end assign actual diff - # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid - # We just use the one mode we should have parsed + # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid. + # We just use the one mode we should have parsed. a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode)) b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode) index.append( @@ -579,7 +587,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non for line in lines.split("\x00:"): if not line: - # The line data is empty, skip + # The line data is empty, skip. continue meta, _, path = line.partition("\x00") path = path.rstrip("\x00") @@ -603,7 +611,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non rename_to = None # NOTE: We cannot conclude from the existence of a blob to change type - # as diffs with the working do not have blobs yet + # as diffs with the working do not have blobs yet. if change_type == "D": b_blob_id = None # Optional[str] deleted_file = True @@ -621,7 +629,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non b_path = b_path_str.encode(defenc) rename_from, rename_to = a_path, b_path elif change_type == "T": - # Nothing to do + # Nothing to do. pass # END add/remove handling diff --git a/git/exc.py b/git/exc.py index 32c371d0b..bfb023fa5 100644 --- a/git/exc.py +++ b/git/exc.py @@ -3,7 +3,8 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ -""" Module containing all exceptions thrown throughout the git package """ + +"""Module containing all exceptions thrown throughout the git package.""" __all__ = [ # Defined in gitdb.exc: @@ -57,7 +58,7 @@ class GitError(Exception): - """Base class for all package exceptions""" + """Base class for all package exceptions.""" class InvalidGitRepositoryError(GitError): @@ -65,7 +66,7 @@ class InvalidGitRepositoryError(GitError): class WorkTreeRepositoryUnsupported(InvalidGitRepositoryError): - """Thrown to indicate we can't handle work tree repositories""" + """Thrown to indicate we can't handle work tree repositories.""" class NoSuchPathError(GitError, OSError): @@ -133,7 +134,7 @@ def __str__(self) -> str: class GitCommandNotFound(CommandError): """Thrown if we cannot find the `git` executable in the PATH or at the path given by - the GIT_PYTHON_GIT_EXECUTABLE environment variable""" + the GIT_PYTHON_GIT_EXECUTABLE environment variable.""" def __init__(self, command: Union[List[str], Tuple[str], str], cause: Union[str, Exception]) -> None: super(GitCommandNotFound, self).__init__(command, cause) @@ -157,15 +158,15 @@ class CheckoutError(GitError): """Thrown if a file could not be checked out from the index as it contained changes. - The .failed_files attribute contains a list of relative paths that failed - to be checked out as they contained changes that did not exist in the index. + The :attr:`failed_files` attribute contains a list of relative paths that failed to + be checked out as they contained changes that did not exist in the index. - The .failed_reasons attribute contains a string informing about the actual + The :attr:`failed_reasons` attribute contains a string informing about the actual cause of the issue. - The .valid_files attribute contains a list of relative paths to files that - were checked out successfully and hence match the version stored in the - index""" + The :attr:`valid_files` attribute contains a list of relative paths to files that + were checked out successfully and hence match the version stored in the index. + """ def __init__( self, @@ -184,18 +185,20 @@ def __str__(self) -> str: class CacheError(GitError): - - """Base for all errors related to the git index, which is called cache internally""" + """Base for all errors related to the git index, which is called cache + internally.""" class UnmergedEntriesError(CacheError): """Thrown if an operation cannot proceed as there are still unmerged - entries in the cache""" + entries in the cache.""" class HookExecutionError(CommandError): - """Thrown if a hook exits with a non-zero exit code. It provides access to the exit code and the string returned - via standard output""" + """Thrown if a hook exits with a non-zero exit code. + + This provides access to the exit code and the string returned via standard output. + """ def __init__( self, @@ -209,7 +212,8 @@ def __init__( class RepositoryDirtyError(GitError): - """Thrown whenever an operation on a repository fails as it has uncommitted changes that would be overwritten""" + """Thrown whenever an operation on a repository fails as it has uncommitted changes + that would be overwritten.""" def __init__(self, repo: "Repo", message: str) -> None: self.repo = repo diff --git a/git/index/__init__.py b/git/index/__init__.py index 96b721f07..f9a534ee7 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -1,4 +1,6 @@ -"""Initialize the index package""" +"""Initialize the index package.""" + # flake8: noqa + from .base import * from .typ import * diff --git a/git/index/base.py b/git/index/base.py index 5da904b4e..4f1b6c469 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -95,12 +95,11 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): - """ - Implements an Index that can be manipulated using a native implementation in - order to save git command function calls wherever possible. + An Index that can be manipulated using a native implementation in order to save git + command function calls wherever possible. - It provides custom merging facilities allowing to merge without actually changing + This provides custom merging facilities allowing to merge without actually changing your index or your working tree. This way you can perform own test-merges based on the index only without having to deal with the working copy. This is useful in case of partial working trees. @@ -115,18 +114,23 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): index.entries[index.entry_key(index_entry_instance)] = index_entry_instance Make sure you use index.write() once you are done manipulating the index directly - before operating on it using the git command""" + before operating on it using the git command. + """ __slots__ = ("repo", "version", "entries", "_extension_data", "_file_path") - _VERSION = 2 # latest version we support - S_IFGITLINK = S_IFGITLINK # a submodule + + _VERSION = 2 # Latest version we support. + + S_IFGITLINK = S_IFGITLINK # A submodule. def __init__(self, repo: "Repo", file_path: Union[PathLike, None] = None) -> None: """Initialize this Index instance, optionally from the given ``file_path``. + If no file_path is given, we will be created from the current index file. If a stream is not given, the stream will be initialized from the current - repository's index on demand.""" + repository's index on demand. + """ self.repo = repo self.version = self._VERSION self._extension_data = b"" @@ -137,7 +141,7 @@ def _set_cache_(self, attr: str) -> None: try: fd = os.open(self._file_path, os.O_RDONLY) except OSError: - # in new repositories, there may be no index, which means we are empty + # In new repositories, there may be no index, which means we are empty. self.entries: Dict[Tuple[PathLike, StageType], IndexEntry] = {} return None # END exception handling @@ -163,18 +167,19 @@ def path(self) -> PathLike: return self._file_path def _delete_entries_cache(self) -> None: - """Safely clear the entries cache so it can be recreated""" + """Safely clear the entries cache so it can be recreated.""" try: del self.entries except AttributeError: - # fails in python 2.6.5 with this exception + # It failed in Python 2.6.5 with AttributeError. + # FIXME: Look into whether we can just remove this except clause now. pass # END exception handling # { Serializable Interface def _deserialize(self, stream: IO) -> "IndexFile": - """Initialize this instance with index values read from the given stream""" + """Initialize this instance with index values read from the given stream.""" self.version, self.entries, self._extension_data, _conten_sha = read_cache(stream) return self @@ -197,7 +202,7 @@ def write( file_path: Union[None, PathLike] = None, ignore_extension_data: bool = False, ) -> None: - """Write the current state to our file path or to the given one + """Write the current state to our file path or to the given one. :param file_path: If None, we will write to our stored file path from which we have @@ -214,12 +219,11 @@ def write( If this data is present in the written index, git-write-tree will instead write the stored/cached tree. Alternatively, use IndexFile.write_tree() to handle this case - automatically - - :return: self # does it? or returns None?""" - # make sure we have our entries read before getting a write lock - # else it would be done when streaming. This can happen - # if one doesn't change the index, but writes it right away + automatically. + """ + # Make sure we have our entries read before getting a write lock. + # Otherwise it would be done when streaming. This can happen if one + # doesn't change the index, but writes it right away. self.entries lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) @@ -232,7 +236,7 @@ def write( lfd.commit() - # make sure we represent what we have written + # Make sure we represent what we have written. if file_path is not None: self._file_path = file_path @@ -242,26 +246,26 @@ def merge_tree(self, rhs: Treeish, base: Union[None, Treeish] = None) -> "IndexF """Merge the given rhs treeish into the current index, possibly taking a common base treeish into account. - As opposed to the :func:`IndexFile.from_tree` method, this allows you to use an already - existing tree as the left side of the merge + As opposed to the :func:`IndexFile.from_tree` method, this allows you to use an + already existing tree as the left side of the merge. :param rhs: - treeish reference pointing to the 'other' side of the merge. + Treeish reference pointing to the 'other' side of the merge. :param base: - optional treeish reference pointing to the common base of 'rhs' and - this index which equals lhs + Optional treeish reference pointing to the common base of 'rhs' and this + index which equals lhs. :return: - self ( containing the merge and possibly unmerged entries in case of - conflicts ) + self (containing the merge and possibly unmerged entries in case of + conflicts) :raise GitCommandError: - If there is a merge conflict. The error will - be raised at the first conflicting path. If you want to have proper - merge resolution to be done by yourself, you have to commit the changed - index ( or make a valid tree from it ) and retry with a three-way - index.from_tree call.""" + If there is a merge conflict. The error will be raised at the first + conflicting path. If you want to have proper merge resolution to be done by + yourself, you have to commit the changed index (or make a valid tree from + it) and retry with a three-way index.from_tree call. + """ # -i : ignore working tree status # --aggressive : handle more merge cases # -m : do an actual merge @@ -276,22 +280,24 @@ def merge_tree(self, rhs: Treeish, base: Union[None, Treeish] = None) -> "IndexF @classmethod def new(cls, repo: "Repo", *tree_sha: Union[str, Tree]) -> "IndexFile": """Merge the given treeish revisions into a new index which is returned. - This method behaves like git-read-tree --aggressive when doing the merge. + + This method behaves like ``git-read-tree --aggressive`` when doing the merge. :param repo: The repository treeish are located in. :param tree_sha: - 20 byte or 40 byte tree sha or tree objects + 20 byte or 40 byte tree sha or tree objects. :return: New IndexFile instance. Its path will be undefined. If you intend to write such a merged Index, supply an alternate file_path - to its 'write' method.""" + to its 'write' method. + """ tree_sha_bytes: List[bytes] = [to_bin_sha(str(t)) for t in tree_sha] base_entries = aggressive_tree_merge(repo.odb, tree_sha_bytes) inst = cls(repo) - # convert to entries dict + # Convert to entries dict. entries: Dict[Tuple[PathLike, int], IndexEntry] = dict( zip( ((e.path, e.stage) for e in base_entries), @@ -305,7 +311,7 @@ def new(cls, repo: "Repo", *tree_sha: Union[str, Tree]) -> "IndexFile": @classmethod def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile": """Merge the given treeish revisions into a new index which is returned. - The original index will remain unaltered + The original index will remain unaltered. :param repo: The repository treeish are located in. @@ -319,10 +325,10 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile one. It behaves like a fast-forward. If 3 Trees are given, a 3-way merge will be performed with the first tree being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree, - tree 3 is the 'other' one + tree 3 is the 'other' one. :param kwargs: - Additional arguments passed to git-read-tree + Additional arguments passed to git-read-tree. :return: New IndexFile instance. It will point to a temporary index location which @@ -336,46 +342,49 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile As the underlying git-read-tree command takes into account the current index, it will be temporarily moved out of the way to assure there are no unsuspected - interferences.""" + interferences. + """ if len(treeish) == 0 or len(treeish) > 3: raise ValueError("Please specify between 1 and 3 treeish, got %i" % len(treeish)) arg_list: List[Union[Treeish, str]] = [] - # ignore that working tree and index possibly are out of date + # Ignore that the working tree and index possibly are out of date. if len(treeish) > 1: - # drop unmerged entries when reading our index and merging + # Drop unmerged entries when reading our index and merging. arg_list.append("--reset") - # handle non-trivial cases the way a real merge does + # Handle non-trivial cases the way a real merge does. arg_list.append("--aggressive") # END merge handling # tmp file created in git home directory to be sure renaming - # works - /tmp/ dirs could be on another device + # works - /tmp/ dirs could be on another device. with ExitStack() as stack: tmp_index = stack.enter_context(tempfile.NamedTemporaryFile(dir=repo.git_dir)) arg_list.append("--index-output=%s" % tmp_index.name) arg_list.extend(treeish) - # move current index out of the way - otherwise the merge may fail - # as it considers existing entries. moving it essentially clears the index. + # Move current index out of the way - otherwise the merge may fail + # as it considers existing entries. Moving it essentially clears the index. # Unfortunately there is no 'soft' way to do it. - # The TemporaryFileSwap assure the original file get put back + # The TemporaryFileSwap assure the original file get put back. stack.enter_context(TemporaryFileSwap(join_path_native(repo.git_dir, "index"))) repo.git.read_tree(*arg_list, **kwargs) index = cls(repo, tmp_index.name) - index.entries # force it to read the file as we will delete the temp-file + index.entries # Force it to read the file as we will delete the temp-file. return index # END index merge handling # UTILITIES @unbare_repo def _iter_expand_paths(self: "IndexFile", paths: Sequence[PathLike]) -> Iterator[PathLike]: - """Expand the directories in list of paths to the corresponding paths accordingly, + """Expand the directories in list of paths to the corresponding paths accordingly. - Note: git will add items multiple times even if a glob overlapped - with manually specified paths or if paths where specified multiple - times - we respect that and do not prune""" + :note: + git will add items multiple times even if a glob overlapped + with manually specified paths or if paths where specified multiple + times - we respect that and do not prune. + """ def raise_exc(e: Exception) -> NoReturn: raise e @@ -389,9 +398,9 @@ def raise_exc(e: Exception) -> NoReturn: # END make absolute path try: - st = os.lstat(abs_path) # handles non-symlinks as well + st = os.lstat(abs_path) # Handles non-symlinks as well. except OSError: - # the lstat call may fail as the path may contain globs as well + # The lstat call may fail as the path may contain globs as well. pass else: if S_ISLNK(st.st_mode): @@ -399,7 +408,7 @@ def raise_exc(e: Exception) -> NoReturn: continue # end check symlink - # if the path is not already pointing to an existing file, resolve globs if possible + # If the path is not already pointing to an existing file, resolve globs if possible. if not os.path.exists(abs_path) and ("?" in abs_path or "*" in abs_path or "[" in abs_path): resolved_paths = glob.glob(abs_path) # not abs_path in resolved_paths: @@ -416,12 +425,12 @@ def raise_exc(e: Exception) -> NoReturn: try: for root, _dirs, files in os.walk(abs_path, onerror=raise_exc): for rela_file in files: - # add relative paths only + # Add relative paths only. yield osp.join(root.replace(rs, ""), rela_file) # END for each file in subdir # END for each subdirectory except OSError: - # was a file or something that could not be iterated + # It was a file or something that could not be iterated. yield abs_path.replace(rs, "") # END path exception handling # END for each path @@ -438,17 +447,20 @@ def _write_path_to_stdin( """Write path to proc.stdin and make sure it processes the item, including progress. :return: stdout string + :param read_from_stdout: if True, proc.stdout will be read after the item - was sent to stdin. In that case, it will return None + was sent to stdin. In that case, it will return None. + :note: There is a bug in git-update-index that prevents it from sending reports just in time. This is why we have a version that tries to read stdout and one which doesn't. In fact, the stdout is not - important as the piped-in files are processed anyway and just in time + important as the piped-in files are processed anyway and just in time. + :note: Newlines are essential here, gits behaviour is somewhat inconsistent on this depending on the version, hence we try our best to deal with newlines carefully. Usually the last newline will not be sent, instead - we will close stdin to break the pipe.""" - + we will close stdin to break the pipe. + """ fprogress(filepath, False, item) rval: Union[None, str] = None @@ -456,7 +468,7 @@ def _write_path_to_stdin( try: proc.stdin.write(("%s\n" % filepath).encode(defenc)) except IOError as e: - # pipe broke, usually because some error happened + # Pipe broke, usually because some error happened. raise fmakeexc() from e # END write exception handling proc.stdin.flush() @@ -475,7 +487,8 @@ def iter_blobs( :param predicate: Function(t) returning True if tuple(stage, Blob) should be yielded by the iterator. A default filter, the BlobFilter, allows you to yield blobs - only if they match a given list of paths.""" + only if they match a given list of paths. + """ for entry in self.entries.values(): blob = entry.to_blob(self.repo) blob.size = entry.size @@ -489,8 +502,7 @@ def unmerged_blobs(self) -> Dict[PathLike, List[Tuple[StageType, Blob]]]: :return: Dict(path : list( tuple( stage, Blob, ...))), being a dictionary associating a path in the index with a list containing - sorted stage/blob pairs - + sorted stage/blob pairs. :note: Blobs that have been removed in one side simply do not exist in the @@ -512,19 +524,21 @@ def entry_key(cls, *entry: Union[BaseIndexEntry, PathLike, StageType]) -> Tuple[ return entry_key(*entry) def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": - """Resolve the blobs given in blob iterator. This will effectively remove the - index entries of the respective path at all non-null stages and add the given - blob as new stage null blob. + """Resolve the blobs given in blob iterator. + + This will effectively remove the index entries of the respective path at all + non-null stages and add the given blob as new stage null blob. For each path there may only be one blob, otherwise a ValueError will be raised claiming the path is already at stage 0. :raise ValueError: if one of the blobs already existed at stage 0 + :return: self :note: You will have to write the index manually once you are done, i.e. - index.resolve_blobs(blobs).write() + ``index.resolve_blobs(blobs).write()``. """ for blob in iter_blobs: stage_null_key = (blob.path, 0) @@ -532,7 +546,7 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": raise ValueError("Path %r already exists at stage 0" % str(blob.path)) # END assert blob is not stage 0 already - # delete all possible stages + # Delete all possible stages. for stage in (1, 2, 3): try: del self.entries[(blob.path, stage)] @@ -550,34 +564,40 @@ def update(self) -> "IndexFile": """Reread the contents of our index file, discarding all cached information we might have. - :note: This is a possibly dangerious operations as it will discard your changes - to index.entries - :return: self""" + :note: This is a possibly dangerous operations as it will discard your changes + to index.entries. + + :return: self + """ self._delete_entries_cache() - # allows to lazily reread on demand + # Allows to lazily reread on demand. return self def write_tree(self) -> Tree: - """Writes this index to a corresponding Tree object into the repository's + """Write this index to a corresponding Tree object into the repository's object database and return it. - :return: Tree object representing this index + :return: Tree object representing this index. + :note: The tree will be written even if one or more objects the tree refers to does not yet exist in the object database. This could happen if you added Entries to the index directly. + :raise ValueError: if there are no entries in the cache - :raise UnmergedEntriesError:""" - # we obtain no lock as we just flush our contents to disk as tree - # If we are a new index, the entries access will load our data accordingly + + :raise UnmergedEntriesError: + """ + # We obtain no lock as we just flush our contents to disk as tree. + # If we are a new index, the entries access will load our data accordingly. mdb = MemoryDB() entries = self._entries_sorted() binsha, tree_items = write_tree_from_cache(entries, mdb, slice(0, len(entries))) - # copy changed trees only + # Copy changed trees only. mdb.stream_copy(mdb.sha_iter(), self.repo.odb) - # note: additional deserialization could be saved if write_tree_from_cache - # would return sorted tree entries + # Note: Additional deserialization could be saved if write_tree_from_cache + # would return sorted tree entries. root_tree = Tree(self.repo, binsha, path="") root_tree._cache = tree_items return root_tree @@ -630,9 +650,10 @@ def _preprocess_add_items( def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry: """Store file at filepath in the database and return the base index entry Needs the git_working_dir decorator active ! This must be assured in the calling code""" - st = os.lstat(filepath) # handles non-symlinks as well + st = os.lstat(filepath) # Handles non-symlinks as well. if S_ISLNK(st.st_mode): - # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8 + # In PY3, readlink is a string, but we need bytes. + # In PY2, it was just OS encoded bytes, we assumed UTF-8. open_stream: Callable[[], BinaryIO] = lambda: BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) else: open_stream = lambda: open(filepath, "rb") @@ -708,51 +729,54 @@ def add( relative or absolute. - path string - strings denote a relative or absolute path into the repository pointing to - an existing file, i.e. CHANGES, lib/myfile.ext, '/home/gitrepo/lib/myfile.ext'. + Strings denote a relative or absolute path into the repository pointing + to an existing file, e.g., CHANGES, lib/myfile.ext, + '/home/gitrepo/lib/myfile.ext'. - Absolute paths must start with working tree directory of this index's repository - to be considered valid. For example, if it was initialized with a non-normalized path, like - `/root/repo/../repo`, absolute paths to be added must start with `/root/repo/../repo`. + Absolute paths must start with working tree directory of this index's + repository to be considered valid. For example, if it was initialized + with a non-normalized path, like ``/root/repo/../repo``, absolute paths + to be added must start with ``/root/repo/../repo``. Paths provided like this must exist. When added, they will be written into the object database. - PathStrings may contain globs, such as 'lib/__init__*' or can be directories - like 'lib', the latter ones will add all the files within the directory and - subdirectories. + PathStrings may contain globs, such as ``lib/__init__*``. Or they can be + directories like ``lib``, which will add all the files within the + directory and subdirectories. This equals a straight git-add. - They are added at stage 0 + They are added at stage 0. - Blob or Submodule object Blobs are added as they are assuming a valid mode is set. - The file they refer to may or may not exist in the file system, but - must be a path relative to our repository. - - If their sha is null ( 40*0 ), their path must exist in the file system - relative to the git repository as an object will be created from - the data at the path. - The handling now very much equals the way string paths are processed, except that - the mode you have set will be kept. This allows you to create symlinks - by settings the mode respectively and writing the target of the symlink - directly into the file. This equals a default Linux-Symlink which - is not dereferenced automatically, except that it can be created on - filesystems not supporting it as well. + + The file they refer to may or may not exist in the file system, but must + be a path relative to our repository. + + If their sha is null (40*0), their path must exist in the file system + relative to the git repository as an object will be created from the + data at the path. + + The handling now very much equals the way string paths are processed, + except that the mode you have set will be kept. This allows you to + create symlinks by settings the mode respectively and writing the target + of the symlink directly into the file. This equals a default + Linux-Symlink which is not dereferenced automatically, except that it + can be created on filesystems not supporting it as well. Please note that globs or directories are not allowed in Blob objects. - They are added at stage 0 + They are added at stage 0. - BaseIndexEntry or type - Handling equals the one of Blob objects, but the stage may be - explicitly set. Please note that Index Entries require binary sha's. + Handling equals the one of Blob objects, but the stage may be explicitly + set. Please note that Index Entries require binary sha's. :param force: **CURRENTLY INEFFECTIVE** - If True, otherwise ignored or excluded files will be - added anyway. + If True, otherwise ignored or excluded files will be added anyway. As opposed to the git-add command, we enable this flag by default as the API user usually wants the item to be added even though they might be excluded. @@ -766,26 +790,31 @@ def add( in the index already as the index is currently being processed. :param path_rewriter: - Function with signature (string) func(BaseIndexEntry) function returning a path - for each passed entry which is the path to be actually recorded for the - object created from entry.path. This allows you to write an index which - is not identical to the layout of the actual files on your hard-disk. - If not None and ``items`` contain plain paths, these paths will be - converted to Entries beforehand and passed to the path_rewriter. - Please note that entry.path is relative to the git repository. + Function with signature (string) func(BaseIndexEntry) function returning a + path for each passed entry which is the path to be actually recorded for the + object created from entry.path. This allows you to write an index which is + not identical to the layout of the actual files on your hard-disk. If not + None and ``items`` contain plain paths, these paths will be converted to + Entries beforehand and passed to the path_rewriter. Please note that + entry.path is relative to the git repository. :param write: If True, the index will be written once it was altered. Otherwise the changes only exist in memory and are not available to git commands. :param write_extension_data: - If True, extension data will be written back to the index. This can lead to issues in case - it is containing the 'TREE' extension, which will cause the `git commit` command to write an - old tree, instead of a new one representing the now changed index. - This doesn't matter if you use `IndexFile.commit()`, which ignores the `TREE` extension altogether. - You should set it to True if you intend to use `IndexFile.commit()` exclusively while maintaining - support for third-party extensions. Besides that, you can usually safely ignore the built-in - extensions when using GitPython on repositories that are not handled manually at all. + If True, extension data will be written back to the index. This can lead to + issues in case it is containing the 'TREE' extension, which will cause the + `git commit` command to write an old tree, instead of a new one representing + the now changed index. + + This doesn't matter if you use :meth:`IndexFile.commit`, which ignores the + `TREE` extension altogether. You should set it to True if you intend to use + :meth:`IndexFile.commit` exclusively while maintaining support for + third-party extensions. Besides that, you can usually safely ignore the + built-in extensions when using GitPython on repositories that are not + handled manually at all. + All current built-in extensions are listed here: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/technical/index-format.txt @@ -793,18 +822,17 @@ def add( List(BaseIndexEntries) representing the entries just actually added. :raise OSError: - if a supplied Path did not exist. Please note that BaseIndexEntry + If a supplied Path did not exist. Please note that BaseIndexEntry Objects that do not have a null sha will be added even if their paths do not exist. """ - # sort the entries into strings and Entries, Blobs are converted to entries - # automatically - # paths can be git-added, for everything else we use git-update-index + # Sort the entries into strings and Entries. Blobs are converted to entries automatically. + # Paths can be git-added. For everything else we use git-update-index. paths, entries = self._preprocess_add_items(items) entries_added: List[BaseIndexEntry] = [] # This code needs a working tree, therefore we try not to run it unless required. # That way, we are OK on a bare repository as well. - # If there are no paths, the rewriter has nothing to do either + # If there are no paths, the rewriter has nothing to do either. if paths: entries_added.extend(self._entries_for_paths(paths, path_rewriter, fprogress, entries)) @@ -818,7 +846,7 @@ def add( # END null mode should be remove # HANDLE ENTRY OBJECT CREATION - # create objects if required, otherwise go with the existing shas + # Create objects if required, otherwise go with the existing shas. null_entries_indices = [i for i, e in enumerate(entries) if e.binsha == Object.NULL_BIN_SHA] if null_entries_indices: @@ -828,7 +856,7 @@ def handle_null_entries(self: "IndexFile") -> None: null_entry = entries[ei] new_entry = self._store_path(null_entry.path, fprogress) - # update null entry + # Update null entry. entries[ei] = BaseIndexEntry( ( null_entry.mode, @@ -844,15 +872,14 @@ def handle_null_entries(self: "IndexFile") -> None: # END null_entry handling # REWRITE PATHS - # If we have to rewrite the entries, do so now, after we have generated - # all object sha's + # If we have to rewrite the entries, do so now, after we have generated all object sha's. if path_rewriter: for i, e in enumerate(entries): entries[i] = BaseIndexEntry((e.mode, e.binsha, e.stage, path_rewriter(e))) # END for each entry # END handle path rewriting - # just go through the remaining entries and provide progress info + # Just go through the remaining entries and provide progress info. for i, entry in enumerate(entries): progress_sent = i in null_entries_indices if not progress_sent: @@ -864,7 +891,7 @@ def handle_null_entries(self: "IndexFile") -> None: # END if there are base entries # FINALIZE - # add the new entries to this instance + # Add the new entries to this instance. for entry in entries_added: self.entries[(entry.path, 0)] = IndexEntry.from_base(entry) @@ -879,9 +906,9 @@ def _items_to_rela_paths( items: Union[PathLike, Sequence[Union[PathLike, BaseIndexEntry, Blob, Submodule]]], ) -> List[PathLike]: """Returns a list of repo-relative paths from the given items which - may be absolute or relative paths, entries or blobs""" + may be absolute or relative paths, entries or blobs.""" paths = [] - # if string put in list + # If string, put in list. if isinstance(items, (str, os.PathLike)): items = [items] @@ -937,17 +964,18 @@ def remove( List(path_string, ...) list of repository relative paths that have been removed effectively. This is interesting to know in case you have provided a directory or - globs. Paths are relative to the repository.""" + globs. Paths are relative to the repository. + """ args = [] if not working_tree: args.append("--cached") args.append("--") - # preprocess paths + # Preprocess paths. paths = self._items_to_rela_paths(items) removed_paths = self.repo.git.rm(args, paths, **kwargs).splitlines() - # process output to gain proper paths + # Process output to gain proper paths. # rm 'path' return [p[4:-1] for p in removed_paths] @@ -960,14 +988,14 @@ def move( **kwargs: Any, ) -> List[Tuple[str, str]]: """Rename/move the items, whereas the last item is considered the destination of - the move operation. If the destination is a file, the first item ( of two ) + the move operation. If the destination is a file, the first item (of two) must be a file as well. If the destination is a directory, it may be preceded by one or more directories or files. The working tree will be affected in non-bare repositories. :parma items: - Multiple types of items are supported, please see the 'remove' method + Multiple types of items are supported, please see the :meth:`remove` method for reference. :param skip_errors: If True, errors such as ones resulting from missing source files will @@ -981,7 +1009,8 @@ def move( actual destination. Relative to the repository root. :raise ValueError: If only one item was given - :raise GitCommandError: If git could not handle your request""" + :raise GitCommandError: If git could not handle your request + """ args = [] if skip_errors: args.append("-k") @@ -993,13 +1022,13 @@ def move( was_dry_run = kwargs.pop("dry_run", kwargs.pop("n", None)) kwargs["dry_run"] = True - # first execute rename in dryrun so the command tells us what it actually does - # ( for later output ) + # First execute rename in dryrun so the command tells us what it actually does. + # (for later output) out = [] mvlines = self.repo.git.mv(args, paths, **kwargs).splitlines() - # parse result - first 0:n/2 lines are 'checking ', the remaining ones - # are the 'renaming' ones which we parse + # Parse result - first 0:n/2 lines are 'checking ', the remaining ones + # are the 'renaming' ones which we parse. for ln in range(int(len(mvlines) / 2), len(mvlines)): tokens = mvlines[ln].split(" to ") assert len(tokens) == 2, "Too many tokens in %s" % mvlines[ln] @@ -1009,12 +1038,12 @@ def move( out.append((tokens[0][9:], tokens[1])) # END for each line to parse - # either prepare for the real run, or output the dry-run result + # Either prepare for the real run, or output the dry-run result. if was_dry_run: return out # END handle dryrun - # now apply the actual operation + # Now apply the actual operation. kwargs.pop("dry_run") self.repo.git.mv(args, paths, **kwargs) @@ -1031,14 +1060,18 @@ def commit( commit_date: Union[datetime.datetime, str, None] = None, skip_hooks: bool = False, ) -> Commit: - """Commit the current default index file, creating a commit object. - For more information on the arguments, see Commit.create_from_tree(). - - :note: If you have manually altered the .entries member of this instance, - don't forget to write() your changes to disk beforehand. - Passing skip_hooks=True is the equivalent of using `-n` - or `--no-verify` on the command line. - :return: Commit object representing the new commit""" + """Commit the current default index file, creating a Commit object. + + For more information on the arguments, see + :meth:`Commit.create_from_tree `. + + :note: If you have manually altered the :attr:`entries` member of this instance, + don't forget to :meth:`write` your changes to disk beforehand. + Passing ``skip_hooks=True`` is the equivalent of using ``-n`` + or ``--no-verify`` on the command line. + + :return: :class:`Commit` object representing the new commit + """ if not skip_hooks: run_commit_hook("pre-commit", self) @@ -1099,11 +1132,11 @@ def checkout( fprogress: Callable = lambda *args: None, **kwargs: Any, ) -> Union[None, Iterator[PathLike], Sequence[PathLike]]: - """Checkout the given paths or all files from the version known to the index into - the working tree. + """Check out the given paths or all files from the version known to the index + into the working tree. - :note: Be sure you have written pending changes using the ``write`` method - in case you have altered the enties dictionary directly + :note: Be sure you have written pending changes using the :meth:`write` method + in case you have altered the entries dictionary directly. :param paths: If None, all paths in the index will be checked out. Otherwise an iterable @@ -1112,20 +1145,20 @@ def checkout( :param force: If True, existing files will be overwritten even if they contain local modifications. - If False, these will trigger a CheckoutError. + If False, these will trigger a :class:`CheckoutError`. :param fprogress: see :func:`IndexFile.add` for signature and explanation. The provided progress information will contain None as path and item if no explicit paths are given. Otherwise progress information will be send - prior and after a file has been checked out + prior and after a file has been checked out. :param kwargs: - Additional arguments to be passed to git-checkout-index + Additional arguments to be passed to git-checkout-index. :return: iterable yielding paths to files which have been checked out and are - guaranteed to match the version stored in the index + guaranteed to match the version stored in the index. :raise exc.CheckoutError: If at least one file failed to be checked out. This is a summary, @@ -1133,7 +1166,7 @@ def checkout( If one of files or directories do not exist in the index ( as opposed to the original git command who ignores them ). Raise GitCommandError if error lines could not be parsed - this truly is - an exceptional state + an exceptional state. .. note:: The checkout is limited to checking out the files in the index. Files which are not in the index anymore and exist in @@ -1307,9 +1340,9 @@ def reset( :param kwargs: Additional keyword arguments passed to git-reset - .. note:: IndexFile.reset, as opposed to HEAD.reset, will not delete anyfiles + .. note:: IndexFile.reset, as opposed to HEAD.reset, will not delete any files in order to maintain a consistent working tree. Instead, it will just - checkout the files according to their state in the index. + check out the files according to their state in the index. If you want git-reset like behaviour, use *HEAD.reset* instead. :return: self""" @@ -1355,40 +1388,40 @@ def diff( create_patch: bool = False, **kwargs: Any, ) -> git_diff.DiffIndex: - """Diff this index against the working copy or a Tree or Commit object + """Diff this index against the working copy or a Tree or Commit object. - For a documentation of the parameters and return values, see, - Diffable.diff + For a documentation of the parameters and return values, see + :meth:`Diffable.diff `. :note: Will only work with indices that represent the default git index as they have not been initialized with a stream. """ - # only run if we are the default repository index + # Only run if we are the default repository index. if self._file_path != self._index_path(): raise AssertionError("Cannot call %r on indices that do not represent the default git index" % self.diff()) - # index against index is always empty + # Index against index is always empty. if other is self.Index: return git_diff.DiffIndex() - # index against anything but None is a reverse diff with the respective + # Index against anything but None is a reverse diff with the respective # item. Handle existing -R flags properly. Transform strings to the object - # so that we can call diff on it + # so that we can call diff on it. if isinstance(other, str): other = self.repo.rev_parse(other) # END object conversion if isinstance(other, Object): # for Tree or Commit - # invert the existing R flag + # Invert the existing R flag. cur_val = kwargs.get("R", False) kwargs["R"] = not cur_val return other.diff(self.Index, paths, create_patch, **kwargs) # END diff against other item handling - # if other is not None here, something is wrong + # If other is not None here, something is wrong. if other is not None: raise ValueError("other must be None, Diffable.Index, a Tree or Commit, was %r" % other) - # diff against working copy - can be handled by superclass natively + # Diff against working copy - can be handled by superclass natively. return super(IndexFile, self).diff(other, paths, create_patch, **kwargs) diff --git a/git/index/fun.py b/git/index/fun.py index b50f1f465..ace0866f4 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -1,6 +1,5 @@ -# Contains standalone functions to accompany the index implementation and make it -# more versatile -# NOTE: Autodoc hates it if this is a docstring +# Standalone functions to accompany the index implementation and make it more versatile. +# NOTE: Autodoc hates it if this is a docstring. from io import BytesIO from pathlib import Path @@ -56,7 +55,7 @@ # ------------------------------------------------------------------------------------ -S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule +S_IFGITLINK = S_IFLNK | S_IFDIR # A submodule. CE_NAMEMASK_INV = ~CE_NAMEMASK __all__ = ( @@ -81,12 +80,13 @@ def _has_file_extension(path: str) -> str: def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: - """Run the commit hook of the given name. Silently ignores hooks that do not exist. + """Run the commit hook of the given name. Silently ignore hooks that do not exist. :param name: name of hook, like 'pre-commit' :param index: IndexFile instance - :param args: arguments passed to hook file - :raises HookExecutionError:""" + :param args: Arguments passed to hook file + :raises HookExecutionError: + """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): return None @@ -128,7 +128,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: def stat_mode_to_index_mode(mode: int) -> int: """Convert the given mode from a stat call to the corresponding index mode - and return it""" + and return it.""" if S_ISLNK(mode): # symlinks return S_IFLNK if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK: # submodules @@ -142,9 +142,10 @@ def write_cache( extension_data: Union[None, bytes] = None, ShaStreamCls: Type[IndexFileSHA1Writer] = IndexFileSHA1Writer, ) -> None: - """Write the cache represented by entries to a stream + """Write the cache represented by entries to a stream. :param entries: **sorted** list of entries + :param stream: stream to wrap into the AdapterStreamCls - it is used for final output. @@ -152,28 +153,29 @@ def write_cache( while writing to it, before the data is passed on to the wrapped stream :param extension_data: any kind of data to write as a trailer, it must begin - a 4 byte identifier, followed by its size ( 4 bytes )""" - # wrap the stream into a compatible writer + a 4 byte identifier, followed by its size (4 bytes). + """ + # Wrap the stream into a compatible writer. stream_sha = ShaStreamCls(stream) tell = stream_sha.tell write = stream_sha.write - # header + # Header version = 2 write(b"DIRC") write(pack(">LL", version, len(entries))) - # body + # Body for entry in entries: beginoffset = tell() write(entry.ctime_bytes) # ctime write(entry.mtime_bytes) # mtime path_str = str(entry.path) path: bytes = force_bytes(path_str, encoding=defenc) - plen = len(path) & CE_NAMEMASK # path length + plen = len(path) & CE_NAMEMASK # Path length assert plen == len(path), "Path %s too long to fit into index" % entry.path - flags = plen | (entry.flags & CE_NAMEMASK_INV) # clear possible previous values + flags = plen | (entry.flags & CE_NAMEMASK_INV) # Clear possible previous values. write( pack( ">LLLLLL20sH", @@ -192,11 +194,11 @@ def write_cache( write(b"\0" * ((beginoffset + real_size) - tell())) # END for each entry - # write previously cached extensions data + # Write previously cached extensions data. if extension_data is not None: stream_sha.write(extension_data) - # write the sha over the content + # Write the sha over the content. stream_sha.write_sha() @@ -208,14 +210,16 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]: unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2))) version, num_entries = unpacked - # TODO: handle version 3: extended data, see read-cache.c + # TODO: Handle version 3: extended data, see read-cache.c. assert version in (1, 2) return version, num_entries def entry_key(*entry: Union[BaseIndexEntry, PathLike, int]) -> Tuple[PathLike, int]: """:return: Key suitable to be used for the index.entries dictionary - :param entry: One instance of type BaseIndexEntry or the path and the stage""" + + :param entry: One instance of type BaseIndexEntry or the path and the stage + """ # def is_entry_key_tup(entry_key: Tuple) -> TypeGuard[Tuple[PathLike, int]]: # return isinstance(entry_key, tuple) and len(entry_key) == 2 @@ -234,14 +238,15 @@ def entry_key(*entry: Union[BaseIndexEntry, PathLike, int]) -> Tuple[PathLike, i def read_cache( stream: IO[bytes], ) -> Tuple[int, Dict[Tuple[PathLike, int], "IndexEntry"], bytes, bytes]: - """Read a cache file from the given stream + """Read a cache file from the given stream. :return: tuple(version, entries_dict, extension_data, content_sha) - * version is the integer version number - * entries dict is a dictionary which maps IndexEntry instances to a path at a stage - * extension_data is '' or 4 bytes of type + 4 bytes of size + size bytes - * content_sha is a 20 byte sha on all cache file contents""" + * version is the integer version number. + * entries dict is a dictionary which maps IndexEntry instances to a path at a stage. + * extension_data is '' or 4 bytes of type + 4 bytes of size + size bytes. + * content_sha is a 20 byte sha on all cache file contents. + """ version, num_entries = read_header(stream) count = 0 entries: Dict[Tuple[PathLike, int], "IndexEntry"] = {} @@ -259,17 +264,17 @@ def read_cache( real_size = (tell() - beginoffset + 8) & ~7 read((beginoffset + real_size) - tell()) entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size)) - # entry_key would be the method to use, but we safe the effort + # entry_key would be the method to use, but we save the effort. entries[(path, entry.stage)] = entry count += 1 # END for each entry - # the footer contains extension data and a sha on the content so far - # Keep the extension footer,and verify we have a sha in the end + # The footer contains extension data and a sha on the content so far. + # Keep the extension footer,and verify we have a sha in the end. # Extension data format is: - # 4 bytes ID - # 4 bytes length of chunk - # repeated 0 - N times + # 4 bytes ID + # 4 bytes length of chunk + # Repeated 0 - N times extension_data = stream.read(~0) assert ( len(extension_data) > 19 @@ -277,7 +282,7 @@ def read_cache( content_sha = extension_data[-20:] - # truncate the sha in the end as we will dynamically create it anyway + # Truncate the sha in the end as we will dynamically create it anyway. extension_data = extension_data[:-20] return (version, entries, extension_data, content_sha) @@ -287,14 +292,15 @@ def write_tree_from_cache( entries: List[IndexEntry], odb: "GitCmdObjectDB", sl: slice, si: int = 0 ) -> Tuple[bytes, List["TreeCacheTup"]]: """Create a tree from the given sorted list of entries and put the respective - trees into the given object database + trees into the given object database. - :param entries: **sorted** list of IndexEntries - :param odb: object database to store the trees in - :param si: start index at which we should start creating subtrees - :param sl: slice indicating the range we should process on the entries list + :param entries: **Sorted** list of IndexEntries + :param odb: Object database to store the trees in + :param si: Start index at which we should start creating subtrees + :param sl: Slice indicating the range we should process on the entries list :return: tuple(binsha, list(tree_entry, ...)) a tuple of a sha and a list of - tree entries being a tuple of hexsha, mode, name""" + tree entries being a tuple of hexsha, mode, name + """ tree_items: List["TreeCacheTup"] = [] ci = sl.start @@ -307,10 +313,10 @@ def write_tree_from_cache( ci += 1 rbound = entry.path.find("/", si) if rbound == -1: - # its not a tree + # It's not a tree. tree_items.append((entry.binsha, entry.mode, entry.path[si:])) else: - # find common base range + # Find common base range. base = entry.path[si:rbound] xi = ci while xi < end: @@ -322,19 +328,19 @@ def write_tree_from_cache( xi += 1 # END find common base - # enter recursion - # ci - 1 as we want to count our current item as well + # Enter recursion. + # ci - 1 as we want to count our current item as well. sha, _tree_entry_list = write_tree_from_cache(entries, odb, slice(ci - 1, xi), rbound + 1) tree_items.append((sha, S_IFDIR, base)) - # skip ahead + # Skip ahead. ci = xi # END handle bounds # END for each entry - # finally create the tree + # Finally create the tree. sio = BytesIO() - tree_to_stream(tree_items, sio.write) # writes to stream as bytes, but doesn't change tree_items + tree_to_stream(tree_items, sio.write) # Writes to stream as bytes, but doesn't change tree_items. sio.seek(0) istream = odb.store(IStream(str_tree_type, len(sio.getvalue()), sio)) @@ -347,16 +353,18 @@ def _tree_entry_to_baseindexentry(tree_entry: "TreeCacheTup", stage: int) -> Bas def aggressive_tree_merge(odb: "GitCmdObjectDB", tree_shas: Sequence[bytes]) -> List[BaseIndexEntry]: """ - :return: list of BaseIndexEntries representing the aggressive merge of the given + :return: List of BaseIndexEntries representing the aggressive merge of the given trees. All valid entries are on stage 0, whereas the conflicting ones are left on stage 1, 2 or 3, whereas stage 1 corresponds to the common ancestor tree, 2 to our tree and 3 to 'their' tree. - :param tree_shas: 1, 2 or 3 trees as identified by their binary 20 byte shas - If 1 or two, the entries will effectively correspond to the last given tree - If 3 are given, a 3 way merge is performed""" + + :param tree_shas: 1, 2 or 3 trees as identified by their binary 20 byte shas. + If 1 or two, the entries will effectively correspond to the last given tree. + If 3 are given, a 3 way merge is performed. + """ out: List[BaseIndexEntry] = [] - # one and two way is the same for us, as we don't have to handle an existing + # One and two way is the same for us, as we don't have to handle an existing # index, instrea if len(tree_shas) in (1, 2): for entry in traverse_tree_recursive(odb, tree_shas[-1], ""): @@ -368,72 +376,72 @@ def aggressive_tree_merge(odb: "GitCmdObjectDB", tree_shas: Sequence[bytes]) -> if len(tree_shas) > 3: raise ValueError("Cannot handle %i trees at once" % len(tree_shas)) - # three trees + # Three trees. for base, ours, theirs in traverse_trees_recursive(odb, tree_shas, ""): if base is not None: - # base version exists + # Base version exists. if ours is not None: - # ours exists + # Ours exists. if theirs is not None: - # it exists in all branches, if it was changed in both - # its a conflict, otherwise we take the changed version - # This should be the most common branch, so it comes first + # It exists in all branches. Ff it was changed in both + # its a conflict. Otherwise, we take the changed version. + # This should be the most common branch, so it comes first. if (base[0] != ours[0] and base[0] != theirs[0] and ours[0] != theirs[0]) or ( base[1] != ours[1] and base[1] != theirs[1] and ours[1] != theirs[1] ): - # changed by both + # Changed by both. out.append(_tree_entry_to_baseindexentry(base, 1)) out.append(_tree_entry_to_baseindexentry(ours, 2)) out.append(_tree_entry_to_baseindexentry(theirs, 3)) elif base[0] != ours[0] or base[1] != ours[1]: - # only we changed it + # Only we changed it. out.append(_tree_entry_to_baseindexentry(ours, 0)) else: - # either nobody changed it, or they did. In either - # case, use theirs + # Either nobody changed it, or they did. In either + # case, use theirs. out.append(_tree_entry_to_baseindexentry(theirs, 0)) # END handle modification else: if ours[0] != base[0] or ours[1] != base[1]: - # they deleted it, we changed it, conflict + # They deleted it, we changed it, conflict. out.append(_tree_entry_to_baseindexentry(base, 1)) out.append(_tree_entry_to_baseindexentry(ours, 2)) # else: - # we didn't change it, ignore + # # We didn't change it, ignore. # pass # END handle our change # END handle theirs else: if theirs is None: - # deleted in both, its fine - its out + # Deleted in both, its fine - it's out. pass else: if theirs[0] != base[0] or theirs[1] != base[1]: - # deleted in ours, changed theirs, conflict + # Deleted in ours, changed theirs, conflict. out.append(_tree_entry_to_baseindexentry(base, 1)) out.append(_tree_entry_to_baseindexentry(theirs, 3)) # END theirs changed # else: - # theirs didn't change + # # Theirs didn't change. # pass # END handle theirs # END handle ours else: - # all three can't be None + # All three can't be None. if ours is None: - # added in their branch + # Added in their branch. assert theirs is not None out.append(_tree_entry_to_baseindexentry(theirs, 0)) elif theirs is None: - # added in our branch + # Added in our branch. out.append(_tree_entry_to_baseindexentry(ours, 0)) else: - # both have it, except for the base, see whether it changed + # Both have it, except for the base, see whether it changed. if ours[0] != theirs[0] or ours[1] != theirs[1]: out.append(_tree_entry_to_baseindexentry(ours, 2)) out.append(_tree_entry_to_baseindexentry(theirs, 3)) else: - # it was added the same in both + # It was added the same in both. out.append(_tree_entry_to_baseindexentry(ours, 0)) # END handle two items # END handle heads diff --git a/git/index/typ.py b/git/index/typ.py index b2c6c371b..046df6e83 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -1,4 +1,4 @@ -"""Module with additional types used by the index""" +"""Module with additional types used by the index.""" from binascii import b2a_hex from pathlib import Path @@ -33,7 +33,6 @@ class BlobFilter(object): - """ Predicate to be used by iter_blobs allowing to filter only return blobs which match the given list of directories or files. @@ -46,7 +45,7 @@ class BlobFilter(object): def __init__(self, paths: Sequence[PathLike]) -> None: """ :param paths: - tuple or list of paths which are either pointing to directories or + Tuple or list of paths which are either pointing to directories or to files relative to the current repository """ self.paths = paths @@ -84,8 +83,7 @@ class BaseIndexEntryHelper(NamedTuple): class BaseIndexEntry(BaseIndexEntryHelper): - - """Small Brother of an index entry which can be created to describe changes + """Small brother of an index entry which can be created to describe changes done to the index in which case plenty of additional information is not required. As the first 4 data members match exactly to the IndexEntry type, methods @@ -138,25 +136,26 @@ def to_blob(self, repo: "Repo") -> Blob: class IndexEntry(BaseIndexEntry): - """Allows convenient access to IndexEntry data without completely unpacking it. - Attributes usully accessed often are cached in the tuple whereas others are + Attributes usually accessed often are cached in the tuple whereas others are unpacked on demand. - See the properties for a mapping between names and tuple indices.""" + See the properties for a mapping between names and tuple indices. + """ @property def ctime(self) -> Tuple[int, int]: """ :return: Tuple(int_time_seconds_since_epoch, int_nano_seconds) of the - file's creation time""" + file's creation time + """ return cast(Tuple[int, int], unpack(">LL", self.ctime_bytes)) @property def mtime(self) -> Tuple[int, int]: - """See ctime property, but returns modification time""" + """See ctime property, but returns modification time.""" return cast(Tuple[int, int], unpack(">LL", self.mtime_bytes)) @classmethod @@ -164,9 +163,10 @@ def from_base(cls, base: "BaseIndexEntry") -> "IndexEntry": """ :return: Minimal entry as created from the given BaseIndexEntry instance. - Missing values will be set to null-like values + Missing values will be set to null-like values. - :param base: Instance of type BaseIndexEntry""" + :param base: Instance of type :class:`BaseIndexEntry` + """ time = pack(">LL", 0, 0) return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) diff --git a/git/index/util.py b/git/index/util.py index 6cf838f3b..f52b61b4a 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -1,4 +1,5 @@ -"""Module containing index utilities""" +"""Module containing index utilities.""" + from functools import wraps import os import struct @@ -33,7 +34,6 @@ class TemporaryFileSwap(object): - """Utility class moving a file to a temporary location within the same directory and moving it back on to where on object deletion.""" @@ -42,7 +42,7 @@ class TemporaryFileSwap(object): def __init__(self, file_path: PathLike) -> None: self.file_path = file_path self.tmp_file_path = str(self.file_path) + tempfile.mktemp("", "", "") - # it may be that the source does not exist + # It may be that the source does not exist. try: os.rename(self.file_path, self.tmp_file_path) except OSError: @@ -90,9 +90,11 @@ def post_clear_cache_if_not_raised(self: "IndexFile", *args: Any, **kwargs: Any) def default_index(func: Callable[..., _T]) -> Callable[..., _T]: - """Decorator assuring the wrapped method may only run if we are the default - repository index. This is as we rely on git commands that operate - on that index only.""" + """Decorator ensuring the wrapped method may only run if we are the default + repository index. + + This is as we rely on git commands that operate on that index only. + """ @wraps(func) def check_default_index(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: @@ -109,7 +111,7 @@ def check_default_index(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]: """Decorator which changes the current working dir to the one of the git - repository in order to assure relative paths are handled correctly""" + repository in order to ensure relative paths are handled correctly.""" @wraps(func) def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 5910ac58a..2a4a114c7 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -1,7 +1,7 @@ -""" -Import all submodules main classes into the package space -""" +"""Import all submodules' main classes into the package space.""" + # flake8: noqa + import inspect from .base import * @@ -14,11 +14,10 @@ from .tree import * # Fix import dependency - add IndexObject to the util module, so that it can be -# imported by the submodule.base +# imported by the submodule.base. smutil.IndexObject = IndexObject # type: ignore[attr-defined] smutil.Object = Object # type: ignore[attr-defined] del smutil -# must come after submodule was made available - +# Must come after submodule was made available. __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] diff --git a/git/objects/base.py b/git/objects/base.py index 0dab5ccdb..0d88aa185 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -38,8 +38,7 @@ class Object(LazyMixin): - - """Implements an Object which may be Blobs, Trees, Commits and Tags""" + """An Object which may be Blobs, Trees, Commits and Tags.""" NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = b"\0" * 20 @@ -50,7 +49,9 @@ class Object(LazyMixin): dbtyp.str_commit_type, dbtyp.str_tag_type, ) + __slots__ = ("repo", "binsha", "size") + type: Union[Lit_commit_ish, None] = None def __init__(self, repo: "Repo", binsha: bytes): @@ -59,7 +60,8 @@ def __init__(self, repo: "Repo", binsha: bytes): :param repo: repository this object is located in - :param binsha: 20 byte SHA1""" + :param binsha: 20 byte SHA1 + """ super(Object, self).__init__() self.repo = repo self.binsha = binsha @@ -71,14 +73,15 @@ def __init__(self, repo: "Repo", binsha: bytes): @classmethod def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish: """ - :return: New Object instance of a type appropriate to the object type behind - id. The id of the newly created object will be a binsha even though - the input id may have been a Reference or Rev-Spec + :return: New :class:`Object`` instance of a type appropriate to the object type + behind `id`. The id of the newly created object will be a binsha even though + the input id may have been a Reference or Rev-Spec. :param id: reference, rev-spec, or hexsha - :note: This cannot be a __new__ method as it would always call __init__ - with the input id which is not necessarily a binsha.""" + :note: This cannot be a ``__new__`` method as it would always call + :meth:`__init__` with the input id which is not necessarily a binsha. + """ return repo.rev_parse(str(id)) @classmethod @@ -86,9 +89,11 @@ def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish: """ :return: new object instance of a type appropriate to represent the given binary sha1 - :param sha1: 20 byte binary sha1""" + + :param sha1: 20 byte binary sha1 + """ if sha1 == cls.NULL_BIN_SHA: - # the NULL binsha is always the root commit + # The NULL binsha is always the root commit. return get_object_type_by_name(b"commit")(repo, sha1) # END handle special case oinfo = repo.odb.info(sha1) @@ -97,7 +102,7 @@ def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish: return inst def _set_cache_(self, attr: str) -> None: - """Retrieve object information""" + """Retrieve object information.""" if attr == "size": oinfo = self.repo.odb.info(self.binsha) self.size = oinfo.size # type: int @@ -137,28 +142,31 @@ def hexsha(self) -> str: @property def data_stream(self) -> "OStream": - """:return: File Object compatible stream to the uncompressed raw data of the object - :note: returned streams must be read in order""" + """ + :return: File Object compatible stream to the uncompressed raw data of the object + + :note: Returned streams must be read in order. + """ return self.repo.odb.stream(self.binsha) def stream_data(self, ostream: "OStream") -> "Object": - """Writes our data directly to the given output stream + """Write our data directly to the given output stream. :param ostream: File object compatible stream object. - :return: self""" + :return: self + """ istream = self.repo.odb.stream(self.binsha) stream_copy(istream, ostream) return self class IndexObject(Object): - - """Base for all objects that can be part of the index file , namely Tree, Blob and - SubModule objects""" + """Base for all objects that can be part of the index file, namely Tree, Blob and + SubModule objects.""" __slots__ = ("path", "mode") - # for compatibility with iterable lists + # For compatibility with iterable lists. _id_attribute_ = "path" def __init__( @@ -168,19 +176,20 @@ def __init__( mode: Union[None, int] = None, path: Union[None, PathLike] = None, ) -> None: - """Initialize a newly instanced IndexObject + """Initialize a newly instanced IndexObject. - :param repo: is the Repo we are located in - :param binsha: 20 byte sha1 + :param repo: The :class:`Repo ` we are located in. + :param binsha: 20 byte sha1. :param mode: - is the stat compatible file mode as int, use the stat module - to evaluate the information + The stat compatible file mode as int, use the :mod:`stat` module to evaluate + the information. :param path: - is the path to the file in the file system, relative to the git repository root, i.e. - file.ext or folder/other.ext + The path to the file in the file system, relative to the git repository + root, like ``file.ext`` or ``folder/other.ext``. :note: - Path may not be set of the index object has been created directly as it cannot - be retrieved without knowing the parent tree.""" + Path may not be set if the index object has been created directly, as it + cannot be retrieved without knowing the parent tree. + """ super(IndexObject, self).__init__(repo, binsha) if mode is not None: self.mode = mode @@ -191,7 +200,8 @@ def __hash__(self) -> int: """ :return: Hash of our path as index items are uniquely identifiable by path, not - by their data !""" + by their data! + """ return hash(self.path) def _set_cache_(self, attr: str) -> None: @@ -214,10 +224,11 @@ def name(self) -> str: def abspath(self) -> PathLike: R""" :return: - Absolute path to this index object in the file system ( as opposed to the - .path field which is a path relative to the git repository ). + Absolute path to this index object in the file system (as opposed to the + :attr:`path` field which is a path relative to the git repository). - The returned path will be native to the system and contains '\' on windows.""" + The returned path will be native to the system and contains '\' on Windows. + """ if self.repo.working_tree_dir is not None: return join_path_native(self.repo.working_tree_dir, self.path) else: diff --git a/git/objects/blob.py b/git/objects/blob.py index 96ce486f5..f0d3181c2 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -3,6 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + from mimetypes import guess_type from . import base @@ -12,13 +13,12 @@ class Blob(base.IndexObject): - - """A Blob encapsulates a git blob object""" + """A Blob encapsulates a git blob object.""" DEFAULT_MIME_TYPE = "text/plain" type: Literal["blob"] = "blob" - # valid blob modes + # Valid blob modes executable_mode = 0o100755 file_mode = 0o100644 link_mode = 0o120000 @@ -29,7 +29,9 @@ class Blob(base.IndexObject): def mime_type(self) -> str: """ :return: String describing the mime type of this file (based on the filename) - :note: Defaults to 'text/plain' in case the actual file type is unknown.""" + + :note: Defaults to 'text/plain' in case the actual file type is unknown. + """ guesses = None if self.path: guesses = guess_type(str(self.path)) diff --git a/git/objects/commit.py b/git/objects/commit.py index fd65fa1e4..29123a44f 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -3,6 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import datetime import re from subprocess import Popen, PIPE @@ -66,7 +67,7 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): value on demand only if it involves calling the git binary.""" # ENVIRONMENT VARIABLES - # read when creating new commits + # Read when creating new commits. env_author_date = "GIT_AUTHOR_DATE" env_committer_date = "GIT_COMMITTER_DATE" @@ -113,36 +114,38 @@ def __init__( be implicitly set on first query. :param binsha: 20 byte sha1 - :param parents: tuple( Commit, ... ) - is a tuple of commit ids or actual Commits + :param parents: tuple(Commit, ...) + A tuple of commit ids or actual Commits :param tree: Tree object :param author: Actor - is the author Actor object + The author Actor object :param authored_date: int_seconds_since_epoch - is the authored DateTime - use time.gmtime() to convert it into a + The authored DateTime - use time.gmtime() to convert it into a different format :param author_tz_offset: int_seconds_west_of_utc - is the timezone that the authored_date is in + The timezone that the authored_date is in :param committer: Actor - is the committer string + The committer string :param committed_date: int_seconds_since_epoch - is the committed DateTime - use time.gmtime() to convert it into a + The committed DateTime - use time.gmtime() to convert it into a different format :param committer_tz_offset: int_seconds_west_of_utc - is the timezone that the committed_date is in + The timezone that the committed_date is in :param message: string - is the commit message + The commit message :param encoding: string - encoding of the message, defaults to UTF-8 + Encoding of the message, defaults to UTF-8 :param parents: List or tuple of Commit objects which are our parent(s) in the commit dependency graph + :return: git.Commit :note: Timezone information is in the same format and in the same sign as what time.altzone returns. The sign is inverted compared to git's - UTC timezone.""" + UTC timezone. + """ super(Commit, self).__init__(repo, binsha) self.binsha = binsha if tree is not None: @@ -211,7 +214,7 @@ def replace(self, **kwargs: Any) -> "Commit": def _set_cache_(self, attr: str) -> None: if attr in Commit.__slots__: - # read the data in a chunk, its faster - then provide a file wrapper + # Read the data in a chunk, its faster - then provide a file wrapper. _binsha, _typename, self.size, stream = self.repo.odb.stream(self.binsha) self._deserialize(BytesIO(stream.read())) else: @@ -235,17 +238,19 @@ def summary(self) -> Union[str, bytes]: return self.message.split(b"\n", 1)[0] def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> int: - """Count the number of commits reachable from this commit + """Count the number of commits reachable from this commit. :param paths: - is an optional path or a list of paths restricting the return value - to commits actually containing the paths + An optional path or a list of paths restricting the return value + to commits actually containing the paths. :param kwargs: Additional options to be passed to git-rev-list. They must not alter - the output style of the command, or parsing will yield incorrect results - :return: int defining the number of reachable commits""" - # yes, it makes a difference whether empty paths are given or not in our case + the output style of the command, or parsing will yield incorrect results. + + :return: An int defining the number of reachable commits + """ + # Yes, it makes a difference whether empty paths are given or not in our case # as the empty paths version will ignore merge commits for some reason. if paths: return len(self.repo.git.rev_list(self.hexsha, "--", paths, **kwargs).splitlines()) @@ -256,7 +261,8 @@ def name_rev(self) -> str: """ :return: String describing the commits hex sha based on the closest Reference. - Mostly useful for UI purposes""" + Mostly useful for UI purposes + """ return self.repo.git.name_rev(self) @classmethod @@ -269,23 +275,29 @@ def iter_items( ) -> Iterator["Commit"]: """Find all commits matching the given criteria. - :param repo: is the Repo - :param rev: revision specifier, see git-rev-parse for viable options + :param repo: The Repo + + :param rev: Revision specifier, see git-rev-parse for viable options. + :param paths: - is an optional path or list of paths, if set only Commits that include the path - or paths will be considered + An optional path or list of paths, if set only Commits that include the path + or paths will be considered. + :param kwargs: - optional keyword arguments to git rev-list where - ``max_count`` is the maximum number of commits to fetch - ``skip`` is the number of commits to skip - ``since`` all commits since i.e. '1970-01-01' - :return: iterator yielding Commit items""" + Optional keyword arguments to ``git rev-list`` where: + + * ``max_count`` is the maximum number of commits to fetch + * ``skip`` is the number of commits to skip + * ``since`` all commits since e.g. '1970-01-01' + + :return: Iterator yielding :class:`Commit` items. + """ if "pretty" in kwargs: raise ValueError("--pretty cannot be used as parsing expects single sha's only") # END handle pretty - # use -- in any case, to prevent possibility of ambiguous arguments - # see https://github.com/gitpython-developers/GitPython/issues/264 + # Use -- in all cases, to prevent possibility of ambiguous arguments. + # See https://github.com/gitpython-developers/GitPython/issues/264. args_list: List[PathLike] = ["--"] @@ -309,7 +321,8 @@ def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs Optional path or list of paths limiting the Commits to those that contain at least one of the paths :param kwargs: All arguments allowed by git-rev-list - :return: Iterator yielding Commit objects which are parents of self""" + :return: Iterator yielding Commit objects which are parents of self + """ # skip ourselves skip = kwargs.get("skip", 1) if skip == 0: # skip ourselves @@ -323,7 +336,8 @@ def stats(self) -> Stats: """Create a git stat from changes between this commit and its first parent or from all changes done if this is the very first commit. - :return: git.Stats""" + :return: git.Stats + """ if not self.parents: text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True) text2 = "" @@ -339,17 +353,18 @@ def stats(self) -> Stats: def trailers(self) -> Dict[str, str]: """Get the trailers of the message as a dictionary - :note: This property is deprecated, please use either ``Commit.trailers_list`` or ``Commit.trailers_dict``. + :note: This property is deprecated, please use either ``Commit.trailers_list`` + or ``Commit.trailers_dict``. :return: - Dictionary containing whitespace stripped trailer information. - Only contains the latest instance of each trailer key. + Dictionary containing whitespace stripped trailer information. Only contains + the latest instance of each trailer key. """ return {k: v[0] for k, v in self.trailers_dict.items()} @property def trailers_list(self) -> List[Tuple[str, str]]: - """Get the trailers of the message as a list + """Get the trailers of the message as a list. Git messages can contain trailer information that are similar to RFC 822 e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). @@ -399,7 +414,7 @@ def trailers_list(self) -> List[Tuple[str, str]]: @property def trailers_dict(self) -> Dict[str, List[str]]: - """Get the trailers of the message as a dictionary + """Get the trailers of the message as a dictionary. Git messages can contain trailer information that are similar to RFC 822 e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). @@ -440,12 +455,14 @@ def trailers_dict(self) -> Dict[str, List[str]]: @classmethod def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, IO]) -> Iterator["Commit"]: - """Parse out commit information into a list of Commit objects + """Parse out commit information into a list of Commit objects. + We expect one-line per commit, and parse the actual commit information directly - from our lighting fast object database + from our lighting fast object database. :param proc: git-rev-list process instance - one sha per line - :return: iterator returning Commit objects""" + :return: iterator supplying :class:`Commit` objects + """ # def is_proc(inp) -> TypeGuard[Popen]: # return hasattr(proc_or_stream, 'wait') and not hasattr(proc_or_stream, 'readline') @@ -468,15 +485,16 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, break hexsha = line.strip() if len(hexsha) > 40: - # split additional information, as returned by bisect for instance + # Split additional information, as returned by bisect for instance. hexsha, _ = line.split(None, 1) # END handle extra info assert len(hexsha) == 40, "Invalid line: %s" % hexsha yield cls(repo, hex_to_bin(hexsha)) # END for each line in stream + # TODO: Review this - it seems process handling got a bit out of control - # due to many developers trying to fix the open file handles issue + # due to many developers trying to fix the open file handles issue. if hasattr(proc_or_stream, "wait"): proc_or_stream = cast(Popen, proc_or_stream) finalize_process(proc_or_stream) @@ -497,38 +515,38 @@ def create_from_tree( """Commit the given tree, creating a commit object. :param repo: Repo object the commit should be part of - :param tree: Tree object or hex or bin sha - the tree of the new commit - :param message: Commit message. It may be an empty string if no message is provided. - It will be converted to a string , in any case. + :param tree: Tree object or hex or bin sha. The tree of the new commit. + :param message: Commit message. It may be an empty string if no message is + provided. It will be converted to a string, in any case. :param parent_commits: - Optional Commit objects to use as parents for the new commit. + Optional :class:`Commit` objects to use as parents for the new commit. If empty list, the commit will have no parents at all and become a root commit. - If None , the current head commit will be the parent of the - new commit object + If None, the current head commit will be the parent of the + new commit object. :param head: If True, the HEAD will be advanced to the new commit automatically. - Else the HEAD will remain pointing on the previous commit. This could + Otherwise the HEAD will remain pointing on the previous commit. This could lead to undesired results when diffing files. :param author: The name of the author, optional. If unset, the repository configuration is used to obtain this value. :param committer: The name of the committer, optional. If unset, the repository configuration is used to obtain this value. - :param author_date: The timestamp for the author field - :param commit_date: The timestamp for the committer field + :param author_date: The timestamp for the author field. + :param commit_date: The timestamp for the committer field. - :return: Commit object representing the new commit + :return: Commit object representing the new commit. :note: Additional information about the committer and Author are taken from the environment or from the git configuration, see git-commit-tree for - more information""" + more information. + """ if parent_commits is None: try: parent_commits = [repo.head.commit] except ValueError: - # empty repositories have no head commit + # Empty repositories have no head commit. parent_commits = [] # END handle parent commits else: @@ -538,11 +556,10 @@ def create_from_tree( # end check parent commit types # END if parent commits are unset - # retrieve all additional information, create a commit object, and - # serialize it + # Retrieve all additional information, create a commit object, and serialize it. # Generally: - # * Environment variables override configuration values - # * Sensible defaults are set according to the git documentation + # * Environment variables override configuration values. + # * Sensible defaults are set according to the git documentation. # COMMITTER AND AUTHOR INFO cr = repo.config_reader() @@ -574,14 +591,14 @@ def create_from_tree( committer_time, committer_offset = unix_time, offset # END set committer time - # assume utf8 encoding + # Assume UTF-8 encoding. enc_section, enc_option = cls.conf_encoding.split(".") conf_encoding = cr.get_value(enc_section, enc_option, cls.default_encoding) if not isinstance(conf_encoding, str): raise TypeError("conf_encoding could not be coerced to str") - # if the tree is no object, make sure we create one - otherwise - # the created commit object is invalid + # If the tree is no object, make sure we create one - otherwise + # the created commit object is invalid. if isinstance(tree, str): tree = repo.tree(tree) # END tree conversion @@ -605,15 +622,15 @@ def create_from_tree( new_commit.binsha = cls._calculate_sha_(repo, new_commit) if head: - # need late import here, importing git at the very beginning throws - # as well ... + # Need late import here, importing git at the very beginning throws + # as well... import git.refs try: repo.head.set_commit(new_commit, logmsg=message) except ValueError: - # head is not yet set to the ref our HEAD points to - # Happens on first commit + # head is not yet set to the ref our HEAD points to. + # Happens on first commit. master = git.refs.Head.create( repo, repo.head.ref, @@ -651,7 +668,7 @@ def _serialize(self, stream: BytesIO) -> "Commit": ).encode(self.encoding) ) - # encode committer + # Encode committer. aname = c.name write( ( @@ -679,7 +696,7 @@ def _serialize(self, stream: BytesIO) -> "Commit": write(b"\n") - # write plain bytes, be sure its encoded according to our encoding + # Write plain bytes, be sure its encoded according to our encoding. if isinstance(self.message, str): write(self.message.encode(self.encoding)) else: @@ -703,11 +720,11 @@ def _deserialize(self, stream: BytesIO) -> "Commit": # END for each parent line self.parents = tuple(self.parents) - # we don't know actual author encoding before we have parsed it, so keep the lines around + # We don't know actual author encoding before we have parsed it, so keep the lines around. author_line = next_line committer_line = readline() - # we might run into one or more mergetag blocks, skip those for now + # We might run into one or more mergetag blocks, skip those for now. next_line = readline() while next_line.startswith(b"mergetag "): next_line = readline() @@ -715,12 +732,11 @@ def _deserialize(self, stream: BytesIO) -> "Commit": next_line = readline() # end skip mergetags - # now we can have the encoding line, or an empty line followed by the optional - # message. + # Now we can have the encoding line, or an empty line followed by the optional message. self.encoding = self.default_encoding self.gpgsig = "" - # read headers + # Read headers. enc = next_line buf = enc.strip() while buf: @@ -743,8 +759,8 @@ def _deserialize(self, stream: BytesIO) -> "Commit": if is_next_header: continue buf = readline().strip() - # decode the authors name + # Decode the author's name. try: ( self.author, @@ -774,8 +790,8 @@ def _deserialize(self, stream: BytesIO) -> "Commit": ) # END handle author's encoding - # a stream from our data simply gives us the plain message - # The end of our message stream is marked with a newline that we strip + # A stream from our data simply gives us the plain message. + # The end of our message stream is marked with a newline that we strip. self.message = stream.read() try: self.message = self.message.decode(self.encoding, "replace") @@ -796,6 +812,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": def co_authors(self) -> List[Actor]: """ Search the commit message for any co-authors of this commit. + Details on co-authors: https://github.blog/2018-01-29-commit-together-with-co-authors/ :return: List of co-authors for this commit (as Actor objects). diff --git a/git/objects/fun.py b/git/objects/fun.py index 043eec721..7756154be 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -1,4 +1,5 @@ -"""Module with functions which are supposed to be as fast as possible""" +"""Module with functions which are supposed to be as fast as possible.""" + from stat import S_ISDIR @@ -36,12 +37,13 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer"], Union[int, None]]) -> None: - """Write the give list of entries into a stream using its write method + """Write the given list of entries into a stream using its write method. :param entries: **sorted** list of tuples with (binsha, mode, name) - :param write: write method which takes a data string""" + :param write: write method which takes a data string + """ ord_zero = ord("0") - bit_mask = 7 # 3 bits set + bit_mask = 7 # 3 bits set. for binsha, mode, name in entries: mode_str = b"" @@ -49,16 +51,16 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer mode_str = bytes([((mode >> (i * 3)) & bit_mask) + ord_zero]) + mode_str # END for each 8 octal value - # git slices away the first octal if its zero + # git slices away the first octal if it's zero. if mode_str[0] == ord_zero: mode_str = mode_str[1:] # END save a byte - # here it comes: if the name is actually unicode, the replacement below + # Here it comes: If the name is actually unicode, the replacement below # will not work as the binsha is not part of the ascii unicode encoding - - # hence we must convert to an utf8 string for it to work properly. + # hence we must convert to an UTF-8 string for it to work properly. # According to my tests, this is exactly what git does, that is it just - # takes the input literally, which appears to be utf8 on linux. + # takes the input literally, which appears to be UTF-8 on linux. if isinstance(name, str): name_bytes = name.encode(defenc) else: @@ -80,32 +82,32 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: while i < len_data: mode = 0 - # read mode - # Some git versions truncate the leading 0, some don't - # The type will be extracted from the mode later + # Read Mode + # Some git versions truncate the leading 0, some don't. + # The type will be extracted from the mode later. while data[i] != space_ord: - # move existing mode integer up one level being 3 bits - # and add the actual ordinal value of the character + # Move existing mode integer up one level being 3 bits + # and add the actual ordinal value of the character. mode = (mode << 3) + (data[i] - ord_zero) i += 1 # END while reading mode - # byte is space now, skip it + # Byte is space now, skip it. i += 1 - # parse name, it is NULL separated + # Parse name, it is NULL separated. ns = i while data[i] != 0: i += 1 # END while not reached NULL - # default encoding for strings in git is utf8 - # Only use the respective unicode object if the byte stream was encoded + # Default encoding for strings in git is UTF-8. + # Only use the respective unicode object if the byte stream was encoded. name_bytes = data[ns:i] name = safe_decode(name_bytes) - # byte is NULL, get next 20 + # Byte is NULL, get next 20. i += 1 sha = data[i : i + 20] i = i + 20 @@ -115,10 +117,11 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: bool, start_at: int) -> EntryTupOrNone: - """return data entry matching the given name and tree mode - or None. - Before the item is returned, the respective data item is set - None in the tree_data list to mark it done""" + """Return data entry matching the given name and tree mode or None. + + Before the item is returned, the respective data item is set None in the + tree_data list to mark it done. + """ try: item = tree_data[start_at] @@ -148,7 +151,7 @@ def _to_full_path(item: EntryTup, path_prefix: str) -> EntryTup: def _to_full_path(item: EntryTupOrNone, path_prefix: str) -> EntryTupOrNone: - """Rebuild entry with given path prefix""" + """Rebuild entry with given path prefix.""" if not item: return item return (item[0], item[1], path_prefix + item[2]) @@ -160,17 +163,23 @@ def traverse_trees_recursive( """ :return: list of list with entries according to the given binary tree-shas. The result is encoded in a list - of n tuple|None per blob/commit, (n == len(tree_shas)), where + of n tuple|None per blob/commit, (n == len(tree_shas)), where: + * [0] == 20 byte sha * [1] == mode as int * [2] == path relative to working tree root + The entry tuple is None if the respective blob/commit did not exist in the given tree. + :param tree_shas: iterable of shas pointing to trees. All trees must - be on the same level. A tree-sha may be None in which case None + be on the same level. A tree-sha may be None in which case None. + :param path_prefix: a prefix to be added to the returned paths on this level, - set it '' for the first iteration - :note: The ordering of the returned items will be partially lost""" + set it '' for the first iteration. + + :note: The ordering of the returned items will be partially lost. + """ trees_data: List[List[EntryTupOrNone]] = [] nt = len(tree_shas) @@ -178,7 +187,7 @@ def traverse_trees_recursive( if tree_sha is None: data: List[EntryTupOrNone] = [] else: - # make new list for typing as list invariant + # Make new list for typing as list invariant. data = list(tree_entries_from_data(odb.stream(tree_sha).read())) # END handle muted trees trees_data.append(data) @@ -186,9 +195,9 @@ def traverse_trees_recursive( out: List[Tuple[EntryTupOrNone, ...]] = [] - # find all matching entries and recursively process them together if the match + # Find all matching entries and recursively process them together if the match # is a tree. If the match is a non-tree item, put it into the result. - # Processed items will be set None + # Processed items will be set None. for ti, tree_data in enumerate(trees_data): for ii, item in enumerate(tree_data): if not item: @@ -198,17 +207,17 @@ def traverse_trees_recursive( entries = [None for _ in range(nt)] entries[ti] = item _sha, mode, name = item - is_dir = S_ISDIR(mode) # type mode bits + is_dir = S_ISDIR(mode) # Type mode bits - # find this item in all other tree data items - # wrap around, but stop one before our current index, hence - # ti+nt, not ti+1+nt + # Find this item in all other tree data items. + # Wrap around, but stop one before our current index, hence + # ti+nt, not ti+1+nt. for tio in range(ti + 1, ti + nt): tio = tio % nt entries[tio] = _find_by_name(trees_data[tio], name, is_dir, ii) # END for each other item data - # if we are a directory, enter recursion + # If we are a directory, enter recursion. if is_dir: out.extend( traverse_trees_recursive( @@ -221,11 +230,11 @@ def traverse_trees_recursive( out.append(tuple(_to_full_path(e, path_prefix) for e in entries)) # END handle recursion - # finally mark it done + # Finally mark it done. tree_data[ii] = None # END for each item - # we are done with one tree, set all its data empty + # We are done with one tree, set all its data empty. del tree_data[:] # END for each tree_data chunk return out @@ -233,16 +242,20 @@ def traverse_trees_recursive( def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: str) -> List[EntryTup]: """ - :return: list of entries of the tree pointed to by the binary tree_sha. An entry - has the following format: + :return: list of entries of the tree pointed to by the binary tree_sha. + + An entry has the following format: + * [0] 20 byte sha * [1] mode as int * [2] path relative to the repository - :param path_prefix: prefix to prepend to the front of all returned paths""" + + :param path_prefix: Prefix to prepend to the front of all returned paths. + """ entries = [] data = tree_entries_from_data(odb.stream(tree_sha).read()) - # unpacking/packing is faster than accessing individual items + # Unpacking/packing is faster than accessing individual items. for sha, mode, name in data: if S_ISDIR(mode): entries.extend(traverse_tree_recursive(odb, sha, path_prefix + name + "/")) diff --git a/git/objects/submodule/__init__.py b/git/objects/submodule/__init__.py index 82df59b0d..8edc13be4 100644 --- a/git/objects/submodule/__init__.py +++ b/git/objects/submodule/__init__.py @@ -1,2 +1,2 @@ -# NOTE: Cannot import anything here as the top-level _init_ has to handle -# our dependencies +# NOTE: Cannot import anything here as the top-level __init__ has to handle +# our dependencies. diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 61c300652..24ea5569c 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -57,9 +57,8 @@ class UpdateProgress(RemoteProgress): - """Class providing detailed progress information to the caller who should - derive from it and implement the ``update(...)`` message""" + derive from it and implement the ``update(...)`` message.""" CLONE, FETCH, UPDWKTREE = [1 << x for x in range(RemoteProgress._num_op_codes, RemoteProgress._num_op_codes + 3)] _num_op_codes: int = RemoteProgress._num_op_codes + 3 @@ -78,25 +77,27 @@ class UpdateProgress(RemoteProgress): # mechanism which cause plenty of trouble of the only reason for packages and # modules is refactoring - subpackages shouldn't depend on parent packages class Submodule(IndexObject, TraversableIterableObj): - """Implements access to a git submodule. They are special in that their sha represents a commit in the submodule's repository which is to be checked out at the path of this instance. + The submodule type does not have a string type associated with it, as it exists solely as a marker in the tree and index. - All methods work in bare and non-bare repositories.""" + All methods work in bare and non-bare repositories. + """ _id_attribute_ = "name" k_modules_file = ".gitmodules" k_head_option = "branch" k_head_default = "master" - k_default_mode = stat.S_IFDIR | stat.S_IFLNK # submodules are directories with link-status + k_default_mode = stat.S_IFDIR | stat.S_IFLNK # Submodules are directories with link-status. - # this is a bogus type for base class compatibility + # This is a bogus type for base class compatibility. type: Literal["submodule"] = "submodule" # type: ignore __slots__ = ("_parent_commit", "_url", "_branch_path", "_name", "__weakref__") + _cache_attrs = ("path", "_url", "_branch_path") def __init__( @@ -110,14 +111,17 @@ def __init__( url: Union[str, None] = None, branch_path: Union[PathLike, None] = None, ) -> None: - """Initialize this instance with its attributes. We only document the ones - that differ from ``IndexObject`` + """Initialize this instance with its attributes. + + We only document the parameters that differ + from :class:`IndexObject `. :param repo: Our parent repository :param binsha: binary sha referring to a commit in the remote repository, see url parameter :param parent_commit: see set_parent_commit() :param url: The url to the remote repository which is the submodule - :param branch_path: full (relative) path to ref to checkout when cloning the remote repository""" + :param branch_path: full (relative) path to ref to checkout when cloning the remote repository + """ super(Submodule, self).__init__(repo, binsha, mode, path) self.size = 0 self._parent_commit = parent_commit @@ -165,18 +169,18 @@ def _need_gitfile_submodules(cls, git: Git) -> bool: return git.version_info[:3] >= (1, 7, 5) def __eq__(self, other: Any) -> bool: - """Compare with another submodule""" - # we may only compare by name as this should be the ID they are hashed with - # Otherwise this type wouldn't be hashable + """Compare with another submodule.""" + # We may only compare by name as this should be the ID they are hashed with. + # Otherwise this type wouldn't be hashable. # return self.path == other.path and self.url == other.url and super(Submodule, self).__eq__(other) return self._name == other._name def __ne__(self, other: object) -> bool: - """Compare with another submodule for inequality""" + """Compare with another submodule for inequality.""" return not (self == other) def __hash__(self) -> int: - """Hash this instance using its logical id, not the sha""" + """Hash this instance using its logical id, not the sha.""" return hash(self._name) def __str__(self) -> str: @@ -195,16 +199,19 @@ def __repr__(self) -> str: def _config_parser( cls, repo: "Repo", parent_commit: Union[Commit_ish, None], read_only: bool ) -> SubmoduleConfigParser: - """:return: Config Parser constrained to our submodule in read or write mode - :raise IOError: If the .gitmodules file cannot be found, either locally or in the repository - at the given parent commit. Otherwise the exception would be delayed until the first - access of the config parser""" + """ + :return: Config Parser constrained to our submodule in read or write mode + + :raise IOError: If the .gitmodules file cannot be found, either locally or in + the repository at the given parent commit. Otherwise the exception would be + delayed until the first access of the config parser. + """ parent_matches_head = True if parent_commit is not None: try: parent_matches_head = repo.head.commit == parent_commit except ValueError: - # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref + # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref. pass # end handle parent_commit fp_module: Union[str, BytesIO] @@ -228,7 +235,7 @@ def _config_parser( return SubmoduleConfigParser(fp_module, read_only=read_only) def _clear_cache(self) -> None: - # clear the possibly changed values + """Clear the possibly changed values.""" for name in self._cache_attrs: try: delattr(self, name) @@ -275,14 +282,16 @@ def _clone_repo( allow_unsafe_protocols: bool = False, **kwargs: Any, ) -> "Repo": - """:return: Repo instance of newly cloned repository - :param repo: our parent repository - :param url: url to clone from - :param path: repository - relative path to the submodule checkout location - :param name: canonical of the submodule + """ + :return: Repo instance of newly cloned repository + :param repo: Our parent repository + :param url: URL to clone from + :param path: Repository - relative path to the submodule checkout location + :param name: Canonical name of the submodule :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: additional arguments given to git.clone""" + :param kwargs: Additional arguments given to git.clone + """ module_abspath = cls._module_abspath(repo, path, name) module_checkout_path = module_abspath if cls._need_gitfile_submodules(repo.git): @@ -331,14 +340,16 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: @classmethod def _write_git_file_and_module_config(cls, working_tree_dir: PathLike, module_abspath: PathLike) -> None: - """Writes a .git file containing a(preferably) relative path to the actual git module repository. + """Write a .git file containing a(preferably) relative path to the actual git module repository. + It is an error if the module_abspath cannot be made into a relative path, relative to the working_tree_dir - :note: will overwrite existing files ! + + :note: This will overwrite existing files! :note: as we rewrite both the git file as well as the module configuration, we might fail on the configuration and will not roll back changes done to the git file. This should be a non - issue, but may easily be fixed - if it becomes one - :param working_tree_dir: directory to write the .git file into - :param module_abspath: absolute path to the bare repository + if it becomes one. + :param working_tree_dir: Directory to write the .git file into + :param module_abspath: Absolute path to the bare repository """ git_file = osp.join(working_tree_dir, ".git") rela_path = osp.relpath(module_abspath, start=working_tree_dir) @@ -377,15 +388,15 @@ def add( If the submodule already exists, no matter if the configuration differs from the one provided, the existing submodule will be returned. - :param repo: Repository instance which should receive the submodule - :param name: The name/identifier for the submodule - :param path: repository-relative or absolute path at which the submodule - should be located + :param repo: Repository instance which should receive the submodule. + :param name: The name/identifier for the submodule. + :param path: Repository-relative or absolute path at which the submodule + should be located. It will be created as required during the repository initialization. - :param url: git-clone compatible URL, see git-clone reference for more information + :param url: git-clone compatible URL, see git-clone reference for more information. If None, the repository is assumed to exist, and the url of the first remote is taken instead. This is useful if you want to make an existing - repository a submodule of anotherone. + repository a submodule of another one. :param branch: name of branch at which the submodule should (later) be checked out. The given branch must exist in the remote repository, and will be checked out locally as a tracking branch. @@ -393,24 +404,25 @@ def add( when the checked out branch will be the one the remote HEAD pointed to. The result you get in these situation is somewhat fuzzy, and it is recommended to specify at least 'master' here. - Examples are 'master' or 'feature/new' - :param no_checkout: if True, and if the repository has to be cloned manually, - no checkout will be performed + Examples are 'master' or 'feature/new'. + :param no_checkout: If True, and if the repository has to be cloned manually, + no checkout will be performed. :param depth: Create a shallow clone with a history truncated to the specified number of commits. :param env: Optional dictionary containing the desired environment variables. - Note: Provided variables will be used to update the execution - environment for `git`. If some variable is not specified in `env` - and is defined in `os.environ`, value from `os.environ` will be used. - If you want to unset some variable, consider providing empty string - as its value. - :param clone_multi_options: A list of Clone options. Please see ``git.repo.base.Repo.clone`` - for details. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext + Note: Provided variables will be used to update the execution environment + for ``git``. If some variable is not specified in `env` and is defined in + attr:`os.environ`, the value from attr:`os.environ` will be used. If you + want to unset some variable, consider providing an empty string as its + value. + :param clone_multi_options: A list of Clone options. Please see + :meth:`Repo.clone ` for details. + :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :return: The newly created submodule instance - :note: works atomically, such that no change will be done if the repository - update fails for instance""" + :return: The newly created submodule instance. + :note: Works atomically, such that no change will be done if the repository + update fails for instance. + """ if repo.bare: raise InvalidGitRepositoryError("Cannot add submodules to bare repositories") @@ -418,11 +430,11 @@ def add( path = cls._to_relative_path(repo, path) - # assure we never put backslashes into the url, as some operating systems - # like it ... + # Ensure we never put backslashes into the URL, as some operating systems + # like it... if url is not None: url = to_native_path_linux(url) - # END assure url correctness + # END ensure URL correctness # INSTANTIATE INTERMEDIATE SM sm = cls( @@ -434,13 +446,13 @@ def add( url="invalid-temporary", ) if sm.exists(): - # reretrieve submodule from tree + # Reretrieve submodule from tree. try: sm = repo.head.commit.tree[str(path)] sm._name = name return sm except KeyError: - # could only be in index + # Could only be in index. index = repo.index entry = index.entries[index.entry_key(path, 0)] sm.binsha = entry.binsha @@ -448,7 +460,7 @@ def add( # END handle exceptions # END handle existing - # fake-repo - we only need the functionality on the branch instance + # fake-repo - we only need the functionality on the branch instance. br = git.Head(repo, git.Head.to_full_path(str(branch) or cls.k_head_default)) has_module = sm.module_exists() branch_is_default = branch is None @@ -501,11 +513,11 @@ def add( ) # END verify url - ## See #525 for ensuring git urls in config-files valid under Windows. + ## See #525 for ensuring git URLs in config-files are valid under Windows. url = Git.polish_url(url) # It's important to add the URL to the parent config, to let `git submodule` know. - # otherwise there is a '-' character in front of the submodule listing + # Otherwise there is a '-' character in front of the submodule listing: # a38efa84daef914e4de58d1905a500d8d14aaf45 mymodule (v0.9.0-1-ga38efa8) # -a38efa84daef914e4de58d1905a500d8d14aaf45 submodules/intermediate/one writer: Union[GitConfigParser, SectionConstraint] @@ -513,7 +525,7 @@ def add( with sm.repo.config_writer() as writer: writer.set_value(sm_section(name), "url", url) - # update configuration and index + # Update configuration and index. index = sm.repo.index with sm.config_writer(index=index, write=False) as writer: writer.set_value("url", url) @@ -525,7 +537,7 @@ def add( writer.set_value(cls.k_head_option, br.path) sm._branch_path = br.path - # we deliberately assume that our head matches our index ! + # We deliberately assume that our head matches our index! if mrepo: sm.binsha = mrepo.head.commit.binsha index.add([sm], write=True) @@ -549,40 +561,54 @@ def update( """Update the repository of this submodule to point to the checkout we point at with the binsha of this instance. - :param recursive: if True, we will operate recursively and update child- - modules as well. - :param init: if True, the module repository will be cloned into place if necessary - :param to_latest_revision: if True, the submodule's sha will be ignored during checkout. - Instead, the remote will be fetched, and the local tracking branch updated. - This only works if we have a local tracking branch, which is the case - if the remote repository had a master branch, or of the 'branch' option - was specified for this submodule and the branch existed remotely - :param progress: UpdateProgress instance or None if no progress should be shown - :param dry_run: if True, the operation will only be simulated, but not performed. - All performed operations are read - only + :param recursive: + If True, we will operate recursively and update child modules as well. + :param init: + If True, the module repository will be cloned into place if necessary. + :param to_latest_revision: + If True, the submodule's sha will be ignored during checkout. Instead, the + remote will be fetched, and the local tracking branch updated. This only + works if we have a local tracking branch, which is the case if the remote + repository had a master branch, or of the 'branch' option was specified for + this submodule and the branch existed remotely. + :param progress: + UpdateProgress instance or None if no progress should be shown. + :param dry_run: + If True, the operation will only be simulated, but not performed. + All performed operations are read-only. :param force: - If True, we may reset heads even if the repository in question is dirty. Additinoally we will be allowed - to set a tracking branch which is ahead of its remote branch back into the past or the location of the - remote branch. This will essentially 'forget' commits. - If False, local tracking branches that are in the future of their respective remote branches will simply - not be moved. - :param keep_going: if True, we will ignore but log all errors, and keep going recursively. - Unless dry_run is set as well, keep_going could cause subsequent / inherited errors you wouldn't see - otherwise. - In conjunction with dry_run, it can be useful to anticipate all errors when updating submodules + If True, we may reset heads even if the repository in question is dirty. + Additionally we will be allowed to set a tracking branch which is ahead of + its remote branch back into the past or the location of the remote branch. + This will essentially 'forget' commits. + If False, local tracking branches that are in the future of their respective + remote branches will simply not be moved. + :param keep_going: + If True, we will ignore but log all errors, and keep going recursively. + Unless dry_run is set as well, keep_going could cause subsequent / inherited + errors you wouldn't see otherwise. + In conjunction with dry_run, it can be useful to anticipate all errors when + updating submodules. :param env: Optional dictionary containing the desired environment variables. - Note: Provided variables will be used to update the execution - environment for `git`. If some variable is not specified in `env` - and is defined in `os.environ`, value from `os.environ` will be used. - If you want to unset some variable, consider providing empty string - as its value. - :param clone_multi_options: list of Clone options. Please see ``git.repo.base.Repo.clone`` - for details. Only take effect with `init` option. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :note: does nothing in bare repositories - :note: method is definitely not atomic if recurisve is True - :return: self""" + Note: Provided variables will be used to update the execution environment + for ``git``. If some variable is not specified in `env` and is defined in + attr:`os.environ`, value from attr:`os.environ` will be used. + If you want to unset some variable, consider providing the empty string as + its value. + :param clone_multi_options: + List of Clone options. + Please see :meth:`Repo.clone ` for details. + They only take effect with the `init` option. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ext. + :param allow_unsafe_options: + Allow unsafe options to be used, like --upload-pack. + + :note: Does nothing in bare repositories. + :note: This method is definitely not atomic if `recursive` is True. + + :return: self + """ if self.repo.bare: return self # END pass in bare mode @@ -595,14 +621,14 @@ def update( prefix = "DRY-RUN: " # END handle prefix - # to keep things plausible in dry-run mode + # To keep things plausible in dry-run mode. if dry_run: mrepo = None # END init mrepo try: - # ASSURE REPO IS PRESENT AND UPTODATE - ##################################### + # ENSURE REPO IS PRESENT AND UP-TO-DATE + ####################################### try: mrepo = self.module() rmts = mrepo.remotes @@ -640,7 +666,7 @@ def update( return self # END early abort if init is not allowed - # there is no git-repository yet - but delete empty paths + # There is no git-repository yet - but delete empty paths. checkout_module_abspath = self.abspath if not dry_run and osp.isdir(checkout_module_abspath): try: @@ -652,8 +678,8 @@ def update( # END handle OSError # END handle directory removal - # don't check it out at first - nonetheless it will create a local - # branch according to the remote-HEAD if possible + # Don't check it out at first - nonetheless it will create a local + # branch according to the remote-HEAD if possible. progress.update( BEGIN | CLONE, 0, @@ -682,19 +708,19 @@ def update( ) if not dry_run: - # see whether we have a valid branch to checkout + # See whether we have a valid branch to check out. try: mrepo = cast("Repo", mrepo) - # find a remote which has our branch - we try to be flexible + # Find a remote which has our branch - we try to be flexible. remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) local_branch = mkhead(mrepo, self.branch_path) - # have a valid branch, but no checkout - make sure we can figure - # that out by marking the commit with a null_sha + # Have a valid branch, but no checkout - make sure we can figure + # that out by marking the commit with a null_sha. local_branch.set_object(Object(mrepo, self.NULL_BIN_SHA)) # END initial checkout + branch creation - # make sure HEAD is not detached + # Make sure HEAD is not detached. mrepo.head.set_reference( local_branch, logmsg="submodule: attaching head to %s" % local_branch, @@ -704,21 +730,21 @@ def update( log.warning("Failed to checkout tracking branch %s", self.branch_path) # END handle tracking branch - # NOTE: Have to write the repo config file as well, otherwise - # the default implementation will be offended and not update the repository - # Maybe this is a good way to assure it doesn't get into our way, but - # we want to stay backwards compatible too ... . Its so redundant ! + # NOTE: Have to write the repo config file as well, otherwise the + # default implementation will be offended and not update the repository. + # Maybe this is a good way to ensure it doesn't get into our way, but + # we want to stay backwards compatible too... It's so redundant! with self.repo.config_writer() as writer: writer.set_value(sm_section(self.name), "url", self.url) # END handle dry_run # END handle initialization - # DETERMINE SHAS TO CHECKOUT - ############################ + # DETERMINE SHAS TO CHECK OUT + ############################# binsha = self.binsha hexsha = self.hexsha if mrepo is not None: - # mrepo is only set if we are not in dry-run mode or if the module existed + # mrepo is only set if we are not in dry-run mode or if the module existed. is_detached = mrepo.head.is_detached # END handle dry_run @@ -742,13 +768,13 @@ def update( # END handle detached head # END handle to_latest_revision option - # update the working tree - # handles dry_run + # Update the working tree. + # Handles dry_run. if mrepo is not None and mrepo.head.commit.binsha != binsha: - # We must assure that our destination sha (the one to point to) is in the future of our current head. - # Otherwise, we will reset changes that might have been done on the submodule, but were not yet pushed + # We must ensure that our destination sha (the one to point to) is in the future of our current head. + # Otherwise, we will reset changes that might have been done on the submodule, but were not yet pushed. # We also handle the case that history has been rewritten, leaving no merge-base. In that case - # we behave conservatively, protecting possible changes the user had done + # we behave conservatively, protecting possible changes the user had done. may_reset = True if mrepo.head.commit.binsha != self.NULL_BIN_SHA: base_commit = mrepo.merge_base(mrepo.head.commit, hexsha) @@ -785,7 +811,7 @@ def update( if not dry_run and may_reset: if is_detached: - # NOTE: for now we force, the user is no supposed to change detached + # NOTE: For now we force. The user is not supposed to change detached # submodules anyway. Maybe at some point this becomes an option, to # properly handle user modifications - see below for future options # regarding rebase and merge. @@ -793,7 +819,7 @@ def update( else: mrepo.head.reset(hexsha, index=True, working_tree=True) # END handle checkout - # if we may reset/checkout + # If we may reset/checkout. progress.update( END | UPDWKTREE, 0, @@ -810,7 +836,7 @@ def update( # HANDLE RECURSION ################## if recursive: - # in dry_run mode, the module might not exist + # In dry_run mode, the module might not exist. if mrepo is not None: for submodule in self.iter_items(self.module()): submodule.update( @@ -834,19 +860,19 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = the repository at our current path, changing the configuration, as well as adjusting our index entry accordingly. - :param module_path: the path to which to move our module in the parent repostory's working tree, - given as repository - relative or absolute path. Intermediate directories will be created - accordingly. If the path already exists, it must be empty. - Trailing(back)slashes are removed automatically - :param configuration: if True, the configuration will be adjusted to let + :param module_path: The path to which to move our module in the parent + repository's working tree, given as repository - relative or absolute path. + Intermediate directories will be created accordingly. If the path already + exists, it must be empty. Trailing (back)slashes are removed automatically. + :param configuration: If True, the configuration will be adjusted to let the submodule point to the given path. - :param module: if True, the repository managed by this submodule - will be moved as well. If False, we don't move the submodule's checkout, which may leave - the parent repository in an inconsistent state. + :param module: If True, the repository managed by this submodule + will be moved as well. If False, we don't move the submodule's checkout, + which may leave the parent repository in an inconsistent state. :return: self - :raise ValueError: if the module path existed and was not empty, or was a file + :raise ValueError: If the module path existed and was not empty, or was a file. :note: Currently the method is not atomic, and it could leave the repository - in an inconsistent state if a sub - step fails for some reason + in an inconsistent state if a sub-step fails for some reason. """ if module + configuration < 1: raise ValueError("You must specify to move at least the module or the configuration of the submodule") @@ -871,7 +897,7 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = raise ValueError("Index entry for target path did already exist") # END handle index key already there - # remove existing destination + # Remove existing destination. if module: if osp.exists(module_checkout_abspath): if len(os.listdir(module_checkout_abspath)): @@ -884,13 +910,13 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = os.rmdir(module_checkout_abspath) # END handle link else: - # recreate parent directories - # NOTE: renames() does that now + # Recreate parent directories. + # NOTE: renames() does that now. pass # END handle existence # END handle module - # move the module into place if possible + # Move the module into place if possible. cur_path = self.abspath renamed_module = False if module and osp.exists(cur_path): @@ -903,8 +929,8 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = # end handle git file rewrite # END move physical module - # rename the index entry - have to manipulate the index directly as - # git-mv cannot be used on submodules ... yeah + # Rename the index entry - we have to manipulate the index directly as + # git-mv cannot be used on submodules... yeah. previous_sm_path = self.path try: if configuration: @@ -918,8 +944,8 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = raise InvalidGitRepositoryError("Submodule's entry at %r did not exist" % (self.path)) from e # END handle submodule doesn't exist - # update configuration - with self.config_writer(index=index) as writer: # auto-write + # Update configuration. + with self.config_writer(index=index) as writer: # Auto-write. writer.set_value("path", module_checkout_path) self.path = module_checkout_path # END handle configuration flag @@ -930,7 +956,7 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = raise # END handle undo rename - # Auto-rename submodule if it's name was 'default', that is, the checkout directory + # Auto-rename submodule if it's name was 'default', that is, the checkout directory. if previous_sm_path == self.name: self.rename(module_checkout_path) # end @@ -946,7 +972,7 @@ def remove( dry_run: bool = False, ) -> "Submodule": """Remove this submodule from the repository. This will remove our entry - from the .gitmodules file and the entry in the .git / config file. + from the .gitmodules file and the entry in the .git/config file. :param module: If True, the checked out module we point to will be deleted as well.If that module is currently on a commit outside any branch in the @@ -959,22 +985,23 @@ def remove( :param force: Enforces the deletion of the module even though it contains modifications. This basically enforces a brute-force file system based deletion. - :param configuration: if True, the submodule is deleted from the configuration, - otherwise it isn't. Although this should be enabled most of the times, + :param configuration: If True, the submodule is deleted from the configuration, + otherwise it isn't. Although this should be enabled most of the time, this flag enables you to safely delete the repository of your submodule. - :param dry_run: if True, we will not actually do anything, but throw the errors - we would usually throw + :param dry_run: If True, we will not actually do anything, but throw the errors + we would usually throw. :return: self - :note: doesn't work in bare repositories - :note: doesn't work atomically, as failure to remove any part of the submodule will leave - an inconsistent state - :raise InvalidGitRepositoryError: thrown if the repository cannot be deleted - :raise OSError: if directories or files could not be removed""" + :note: Doesn't work in bare repositories. + :note: Doesn't work atomically, as failure to remove any part of the submodule + will leave an inconsistent state. + :raise InvalidGitRepositoryError: Thrown if the repository cannot be deleted. + :raise OSError: If directories or files could not be removed. + """ if not (module or configuration): raise ValueError("Need to specify to delete at least the module, or the configuration") # END handle parameters - # Recursively remove children of this submodule + # Recursively remove children of this submodule. nc = 0 for csm in self.children(): nc += 1 @@ -982,8 +1009,8 @@ def remove( del csm # end if configuration and not dry_run and nc > 0: - # Assure we don't leave the parent repository in a dirty state, and commit our changes - # It's important for recursive, unforced, deletions to work as expected + # Ensure we don't leave the parent repository in a dirty state, and commit our changes. + # It's important for recursive, unforced, deletions to work as expected. self.module().index.commit("Removed at least one of child-modules of '%s'" % self.name) # end handle recursion @@ -993,9 +1020,9 @@ def remove( mod = self.module() git_dir = mod.git_dir if force: - # take the fast lane and just delete everything in our module path + # Take the fast lane and just delete everything in our module path. # TODO: If we run into permission problems, we have a highly inconsistent - # state. Delete the .git folders last, start with the submodules first + # state. Delete the .git folders last, start with the submodules first. mp = self.abspath method: Union[None, Callable[[PathLike], None]] = None if osp.islink(mp): @@ -1010,7 +1037,7 @@ def remove( method(mp) # END apply deletion method else: - # verify we may delete our module + # Verify we may delete our module. if mod.is_dirty(index=True, working_tree=True, untracked_files=True): raise InvalidGitRepositoryError( "Cannot delete module at %s with any modifications, unless force is specified" @@ -1018,25 +1045,27 @@ def remove( ) # END check for dirt - # figure out whether we have new commits compared to the remotes - # NOTE: If the user pulled all the time, the remote heads might - # not have been updated, so commits coming from the remote look - # as if they come from us. But we stay strictly read-only and - # don't fetch beforehand. + # Figure out whether we have new commits compared to the remotes. + # NOTE: If the user pulled all the time, the remote heads might not have + # been updated, so commits coming from the remote look as if they come + # from us. But we stay strictly read-only and don't fetch beforehand. for remote in mod.remotes: num_branches_with_new_commits = 0 rrefs = remote.refs for rref in rrefs: num_branches_with_new_commits += len(mod.git.cherry(rref)) != 0 # END for each remote ref - # not a single remote branch contained all our commits + # Not a single remote branch contained all our commits. if len(rrefs) and num_branches_with_new_commits == len(rrefs): raise InvalidGitRepositoryError( "Cannot delete module at %s as there are new commits" % mod.working_tree_dir ) # END handle new commits - # have to manually delete references as python's scoping is - # not existing, they could keep handles open ( on windows this is a problem ) + # We have to manually delete some references to allow resources to + # be cleaned up immediately when we are done with them, because + # Python's scoping is no more granular than the whole function (loop + # bodies are not scopes). When the objects stay alive longer, they + # can keep handles open. On Windows, this is a problem. if len(rrefs): del rref # skipcq: PYL-W0631 # END handle remotes @@ -1044,11 +1073,11 @@ def remove( del remote # END for each remote - # finally delete our own submodule + # Finally delete our own submodule. if not dry_run: self._clear_cache() wtd = mod.working_tree_dir - del mod # release file-handles (windows) + del mod # Release file-handles (Windows). import gc gc.collect() @@ -1062,14 +1091,14 @@ def remove( # end handle separate bare repository # END handle module deletion - # void our data not to delay invalid access + # Void our data so as not to delay invalid access. if not dry_run: self._clear_cache() # DELETE CONFIGURATION ###################### if configuration and not dry_run: - # first the index-entry + # First the index-entry. parent_index = self.repo.index try: del parent_index.entries[parent_index.entry_key(self.path, 0)] @@ -1078,8 +1107,8 @@ def remove( # END delete entry parent_index.write() - # now git config - need the config intact, otherwise we can't query - # information anymore + # Now git config - we need the config intact, otherwise we can't query + # information anymore. with self.repo.config_writer() as gcp_writer: gcp_writer.remove_section(sm_section(self.name)) @@ -1095,15 +1124,16 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) contain the .gitmodules blob. :param commit: - Commit-ish reference pointing at the root_tree, or None to always point to the - most recent commit + Commit-ish reference pointing at the root_tree, or None to always point to + the most recent commit :param check: - if True, relatively expensive checks will be performed to verify + If True, relatively expensive checks will be performed to verify validity of the submodule. - :raise ValueError: if the commit's tree didn't contain the .gitmodules blob. + :raise ValueError: If the commit's tree didn't contain the .gitmodules blob. :raise ValueError: - if the parent commit didn't store this submodule under the current path - :return: self""" + If the parent commit didn't store this submodule under the current path. + :return: self + """ if commit is None: self._parent_commit = None return self @@ -1125,9 +1155,9 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) # END handle submodule did not exist # END handle checking mode - # update our sha, it could have changed - # If check is False, we might see a parent-commit that doesn't even contain the submodule anymore. - # in that case, mark our sha as being NULL + # Update our sha, it could have changed. + # If check is False, we might see a parent-commit that doesn't even contain the + # submodule anymore. in that case, mark our sha as being NULL. try: self.binsha = pctree[str(self.path)].binsha except KeyError: @@ -1141,19 +1171,23 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) def config_writer( self, index: Union["IndexFile", None] = None, write: bool = True ) -> SectionConstraint["SubmoduleConfigParser"]: - """:return: a config writer instance allowing you to read and write the data + """ + :return: A config writer instance allowing you to read and write the data belonging to this submodule into the .gitmodules file. - :param index: if not None, an IndexFile instance which should be written. - defaults to the index of the Submodule's parent repository. - :param write: if True, the index will be written each time a configuration + :param index: If not None, an IndexFile instance which should be written. + Defaults to the index of the Submodule's parent repository. + :param write: If True, the index will be written each time a configuration value changes. - :note: the parameters allow for a more efficient writing of the index, + + :note: The parameters allow for a more efficient writing of the index, as you can pass in a modified index on your own, prevent automatic writing, - and write yourself once the whole operation is complete - :raise ValueError: if trying to get a writer on a parent_commit which does not - match the current head commit - :raise IOError: If the .gitmodules file/blob could not be read""" + and write yourself once the whole operation is complete. + + :raise ValueError: If trying to get a writer on a parent_commit which does not + match the current head commit. + :raise IOError: If the .gitmodules file/blob could not be read + """ writer = self._config_parser_constrained(read_only=False) if index is not None: writer.config._index = index @@ -1162,17 +1196,19 @@ def config_writer( @unbare_repo def rename(self, new_name: str) -> "Submodule": - """Rename this submodule - :note: This method takes care of renaming the submodule in various places, such as + """Rename this submodule. + + :note: + This method takes care of renaming the submodule in various places, such as: * $parent_git_dir / config * $working_tree_dir / .gitmodules * (git >= v1.8.0: move submodule repository to new name) - As .gitmodules will be changed, you would need to make a commit afterwards. The changed .gitmodules file - will already be added to the index + As .gitmodules will be changed, you would need to make a commit afterwards. The + changed .gitmodules file will already be added to the index. - :return: this submodule instance + :return: This submodule instance """ if self.name == new_name: return self @@ -1195,7 +1231,7 @@ def rename(self, new_name: str) -> "Submodule": if mod.has_separate_working_tree(): destination_module_abspath = self._module_abspath(self.repo, self.path, new_name) source_dir = mod.git_dir - # Let's be sure the submodule name is not so obviously tied to a directory + # Let's be sure the submodule name is not so obviously tied to a directory. if str(destination_module_abspath).startswith(str(mod.git_dir)): tmp_dir = self._module_abspath(self.repo, self.path, str(uuid.uuid4())) os.renames(source_dir, tmp_dir) @@ -1214,9 +1250,12 @@ def rename(self, new_name: str) -> "Submodule": @unbare_repo def module(self) -> "Repo": - """:return: Repo instance initialized from the repository at our submodule path - :raise InvalidGitRepositoryError: if a repository was not available. This could - also mean that it was not yet initialized""" + """ + :return: Repo instance initialized from the repository at our submodule path + + :raise InvalidGitRepositoryError: If a repository was not available. This could + also mean that it was not yet initialized. + """ module_checkout_abspath = self.abspath try: repo = git.Repo(module_checkout_abspath) @@ -1230,7 +1269,7 @@ def module(self) -> "Repo": # END handle exceptions def module_exists(self) -> bool: - """:return: True if our module exists and is a valid git repository. See module() method""" + """:return: True if our module exists and is a valid git repository. See module() method.""" try: self.module() return True @@ -1241,10 +1280,11 @@ def module_exists(self) -> bool: def exists(self) -> bool: """ :return: True if the submodule exists, False otherwise. Please note that - a submodule may exist ( in the .gitmodules file) even though its module - doesn't exist on disk""" - # keep attributes for later, and restore them if we have no valid data - # this way we do not actually alter the state of the object + a submodule may exist (in the .gitmodules file) even though its module + doesn't exist on disk. + """ + # Keep attributes for later, and restore them if we have no valid data. + # This way we do not actually alter the state of the object. loc = locals() for attr in self._cache_attrs: try: @@ -1252,7 +1292,7 @@ def exists(self) -> bool: loc[attr] = getattr(self, attr) # END if we have the attribute cache except (cp.NoSectionError, ValueError): - # on PY3, this can happen apparently ... don't know why this doesn't happen on PY2 + # On PY3, this can happen apparently... don't know why this doesn't happen on PY2. pass # END for each attr self._clear_cache() @@ -1274,22 +1314,26 @@ def exists(self) -> bool: @property def branch(self) -> "Head": - """:return: The branch instance that we are to checkout - :raise InvalidGitRepositoryError: if our module is not yet checked out""" + """ + :return: The branch instance that we are to checkout + + :raise InvalidGitRepositoryError: If our module is not yet checked out + """ return mkhead(self.module(), self._branch_path) @property def branch_path(self) -> PathLike: """ - :return: full(relative) path as string to the branch we would checkout - from the remote and track""" + :return: Full (relative) path as string to the branch we would checkout + from the remote and track + """ return self._branch_path @property def branch_name(self) -> str: - """:return: the name of the branch, which is the shortest possible branch name""" - # use an instance method, for this we create a temporary Head instance - # which uses a repository that is available at least ( it makes no difference ) + """:return: The name of the branch, which is the shortest possible branch name""" + # Use an instance method, for this we create a temporary Head instance + # which uses a repository that is available at least (it makes no difference). return git.Head(self.repo, self._branch_path).name @property @@ -1299,37 +1343,46 @@ def url(self) -> str: @property def parent_commit(self) -> "Commit_ish": - """:return: Commit instance with the tree containing the .gitmodules file - :note: will always point to the current head's commit if it was not set explicitly""" + """ + :return: Commit instance with the tree containing the .gitmodules file + + :note: Will always point to the current head's commit if it was not set explicitly. + """ if self._parent_commit is None: return self.repo.commit() return self._parent_commit @property def name(self) -> str: - """:return: The name of this submodule. It is used to identify it within the + """ + :return: The name of this submodule. It is used to identify it within the .gitmodules file. - :note: by default, the name is the path at which to find the submodule, but - in git - python it should be a unique identifier similar to the identifiers - used for remotes, which allows to change the path of the submodule - easily + + :note: By default, this is the name is the path at which to find the submodule, + but in GitPython it should be a unique identifier similar to the identifiers + used for remotes, which allows to change the path of the submodule easily. """ return self._name def config_reader(self) -> SectionConstraint[SubmoduleConfigParser]: """ - :return: ConfigReader instance which allows you to qurey the configuration values - of this submodule, as provided by the .gitmodules file - :note: The config reader will actually read the data directly from the repository - and thus does not need nor care about your working tree. - :note: Should be cached by the caller and only kept as long as needed - :raise IOError: If the .gitmodules file/blob could not be read""" + :return: ConfigReader instance which allows you to query the configuration + values of this submodule, as provided by the .gitmodules file. + + :note: The config reader will actually read the data directly from the + repository and thus does not need nor care about your working tree. + + :note: Should be cached by the caller and only kept as long as needed. + + :raise IOError: If the .gitmodules file/blob could not be read. + """ return self._config_parser_constrained(read_only=True) def children(self) -> IterableList["Submodule"]: """ :return: IterableList(Submodule, ...) an iterable list of submodules instances - which are children of this submodule or 0 if the submodule is not checked out""" + which are children of this submodule or 0 if the submodule is not checked out. + """ return self._get_intermediate_items(self) # } END query interface @@ -1344,9 +1397,9 @@ def iter_items( *Args: Any, **kwargs: Any, ) -> Iterator["Submodule"]: - """:return: iterator yielding Submodule instances available in the given repository""" + """:return: Iterator yielding Submodule instances available in the given repository""" try: - pc = repo.commit(parent_commit) # parent commit instance + pc = repo.commit(parent_commit) # Parent commit instance parser = cls._config_parser(repo, pc, read_only=True) except (IOError, BadName): return iter([]) @@ -1361,13 +1414,13 @@ def iter_items( b = str(parser.get(sms, cls.k_head_option)) # END handle optional information - # get the binsha + # Get the binsha. index = repo.index try: - rt = pc.tree # root tree + rt = pc.tree # Root tree sm = rt[p] except KeyError: - # try the index, maybe it was just added + # Try the index, maybe it was just added. try: entry = index.entries[index.entry_key(p, 0)] sm = Submodule(repo, entry.binsha, entry.mode, entry.path) @@ -1378,15 +1431,15 @@ def iter_items( # END handle keyerror # END handle critical error - # Make sure we are looking at a submodule object + # Make sure we are looking at a submodule object. if type(sm) is not git.objects.submodule.base.Submodule: continue - # fill in remaining info - saves time as it doesn't have to be parsed again + # Fill in remaining info - saves time as it doesn't have to be parsed again. sm._name = n if pc != repo.commit(): sm._parent_commit = pc - # end set only if not most recent ! + # end set only if not most recent! sm._branch_path = git.Head.to_full_path(b) sm._url = u diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index d338441ef..b3c06ce9a 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -24,7 +24,7 @@ class RootUpdateProgress(UpdateProgress): - """Utility class which adds more opcodes to the UpdateProgress""" + """Utility class which adds more opcodes to the UpdateProgress.""" REMOVE, PATHCHANGE, BRANCHCHANGE, URLCHANGE = [ 1 << x for x in range(UpdateProgress._num_op_codes, UpdateProgress._num_op_codes + 4) @@ -67,7 +67,7 @@ def __init__(self, repo: "Repo"): ) def _clear_cache(self) -> None: - """May not do anything""" + """May not do anything.""" pass # { Interface @@ -85,37 +85,48 @@ def update( keep_going: bool = False, ) -> "RootModule": """Update the submodules of this repository to the current HEAD commit. + This method behaves smartly by determining changes of the path of a submodules repository, next to changes to the to-be-checked-out commit or the branch to be checked out. This works if the submodules ID does not change. - Additionally it will detect addition and removal of submodules, which will be handled - gracefully. + Additionally it will detect addition and removal of submodules, which will be + handled gracefully. - :param previous_commit: If set to a commit-ish, the commit we should use - as the previous commit the HEAD pointed to before it was set to the commit it points to now. - If None, it defaults to HEAD@{1} otherwise + :param previous_commit: If set to a commit-ish, the commit we should use as the + previous commit the HEAD pointed to before it was set to the commit it + points to now. + If None, it defaults to ``HEAD@{1}`` otherwise :param recursive: if True, the children of submodules will be updated as well - using the same technique - :param force_remove: If submodules have been deleted, they will be forcibly removed. - Otherwise the update may fail if a submodule's repository cannot be deleted as - changes have been made to it (see Submodule.update() for more information) - :param init: If we encounter a new module which would need to be initialized, then do it. - :param to_latest_revision: If True, instead of checking out the revision pointed to - by this submodule's sha, the checked out tracking branch will be merged with the - latest remote branch fetched from the repository's origin. - Unless force_reset is specified, a local tracking branch will never be reset into its past, therefore - the remote branch must be in the future for this to have an effect. - :param force_reset: if True, submodules may checkout or reset their branch even if the repository has - pending changes that would be overwritten, or if the local tracking branch is in the future of the - remote tracking branch and would be reset into its past. - :param progress: RootUpdateProgress instance or None if no progress should be sent - :param dry_run: if True, operations will not actually be performed. Progress messages - will change accordingly to indicate the WOULD DO state of the operation. - :param keep_going: if True, we will ignore but log all errors, and keep going recursively. - Unless dry_run is set as well, keep_going could cause subsequent/inherited errors you wouldn't see - otherwise. - In conjunction with dry_run, it can be useful to anticipate all errors when updating submodules - :return: self""" + using the same technique. + :param force_remove: If submodules have been deleted, they will be forcibly + removed. Otherwise the update may fail if a submodule's repository cannot be + deleted as changes have been made to it. + (See :meth:`Submodule.update ` + for more information.) + :param init: If we encounter a new module which would need to be initialized, + then do it. + :param to_latest_revision: If True, instead of checking out the revision pointed + to by this submodule's sha, the checked out tracking branch will be merged + with the latest remote branch fetched from the repository's origin. + Unless `force_reset` is specified, a local tracking branch will never be + reset into its past, therefore the remote branch must be in the future for + this to have an effect. + :param force_reset: If True, submodules may checkout or reset their branch even + if the repository has pending changes that would be overwritten, or if the + local tracking branch is in the future of the remote tracking branch and + would be reset into its past. + :param progress: :class:`RootUpdateProgress` instance or None if no progress + should be sent. + :param dry_run: If True, operations will not actually be performed. Progress + messages will change accordingly to indicate the WOULD DO state of the + operation. + :param keep_going: If True, we will ignore but log all errors, and keep going + recursively. Unless `dry_run` is set as well, `keep_going` could cause + subsequent/inherited errors you wouldn't see otherwise. + In conjunction with `dry_run`, this can be useful to anticipate all errors + when updating submodules. + :return: self + """ if self.repo.bare: raise InvalidGitRepositoryError("Cannot update submodules in bare repositories") # END handle bare @@ -141,11 +152,11 @@ def update( raise IndexError # END handle initial commit except IndexError: - # in new repositories, there is no previous commit + # In new repositories, there is no previous commit. previous_commit = cur_commit # END exception handling else: - previous_commit = repo.commit(previous_commit) # obtain commit object + previous_commit = repo.commit(previous_commit) # Obtain commit object. # END handle previous commit psms: "IterableList[Submodule]" = self.list_items(repo, parent_commit=previous_commit) @@ -164,8 +175,8 @@ def update( op |= BEGIN # END handle begin - # fake it into thinking its at the current commit to allow deletion - # of previous module. Trigger the cache to be updated before that + # Fake it into thinking its at the current commit to allow deletion + # of previous module. Trigger the cache to be updated before that. progress.update( op, i, @@ -188,7 +199,7 @@ def update( # HANDLE PATH RENAMES ##################### - # url changes + branch changes + # URL changes + branch changes. csms = spsms & ssms len_csms = len(csms) for i, csm in enumerate(csms): @@ -204,7 +215,7 @@ def update( len_csms, prefix + "Moving repository of submodule %r from %s to %s" % (sm.name, psm.abspath, sm.abspath), ) - # move the module to the new path + # Move the module to the new path. if not dry_run: psm.move(sm.path, module=True, configuration=False) # END handle dry_run @@ -220,14 +231,14 @@ def update( # HANDLE URL CHANGE ################### if sm.url != psm.url: - # Add the new remote, remove the old one + # Add the new remote, remove the old one. # This way, if the url just changes, the commits will not - # have to be re-retrieved + # have to be re-retrieved. nn = "__new_origin__" smm = sm.module() rmts = smm.remotes - # don't do anything if we already have the url we search in place + # Don't do anything if we already have the url we search in place. if len([r for r in rmts if r.url == sm.url]) == 0: progress.update( BEGIN | URLCHANGE, @@ -250,7 +261,7 @@ def update( ) # END head is not detached - # now delete the changed one + # Now delete the changed one. rmt_for_deletion = None for remote in rmts: if remote.url == psm.url: @@ -259,17 +270,17 @@ def update( # END if urls match # END for each remote - # if we didn't find a matching remote, but have exactly one, - # we can safely use this one + # If we didn't find a matching remote, but have exactly one, + # we can safely use this one. if rmt_for_deletion is None: if len(rmts) == 1: rmt_for_deletion = rmts[0] else: - # if we have not found any remote with the original url + # If we have not found any remote with the original URL # we may not have a name. This is a special case, - # and its okay to fail here - # Alternatively we could just generate a unique name and leave all - # existing ones in place + # and its okay to fail here. + # Alternatively we could just generate a unique name and + # leave all existing ones in place. raise InvalidGitRepositoryError( "Couldn't find original remote-repo at url %r" % psm.url ) @@ -280,15 +291,15 @@ def update( smm.delete_remote(rmt_for_deletion) # NOTE: Currently we leave tags from the deleted remotes # as well as separate tracking branches in the possibly totally - # changed repository ( someone could have changed the url to - # another project ). At some point, one might want to clean + # changed repository (someone could have changed the url to + # another project). At some point, one might want to clean # it up, but the danger is high to remove stuff the user - # has added explicitly + # has added explicitly. - # rename the new remote back to what it was + # Rename the new remote back to what it was. smr.rename(orig_name) - # early on, we verified that the our current tracking branch + # Early on, we verified that the our current tracking branch # exists in the remote. Now we have to assure that the # sha we point to is still contained in the new remote # tracking branch. @@ -303,10 +314,10 @@ def update( # END for each commit if not found: - # adjust our internal binsha to use the one of the remote - # this way, it will be checked out in the next step + # Adjust our internal binsha to use the one of the remote + # this way, it will be checked out in the next step. # This will change the submodule relative to us, so - # the user will be able to commit the change easily + # the user will be able to commit the change easily. log.warning( "Current sha %s was not contained in the tracking\ branch at the new remote, setting it the the remote's tracking branch", @@ -315,7 +326,7 @@ def update( sm.binsha = rref.commit.binsha # END reset binsha - # NOTE: All checkout is performed by the base implementation of update + # NOTE: All checkout is performed by the base implementation of update. # END handle dry_run progress.update( END | URLCHANGE, @@ -329,8 +340,7 @@ def update( # HANDLE PATH CHANGES ##################### if sm.branch_path != psm.branch_path: - # finally, create a new tracking branch which tracks the - # new remote branch + # Finally, create a new tracking branch which tracks the new remote branch. progress.update( BEGIN | BRANCHCHANGE, i, @@ -342,7 +352,8 @@ def update( if not dry_run: smm = sm.module() smmr = smm.remotes - # As the branch might not exist yet, we will have to fetch all remotes to be sure ... . + # As the branch might not exist yet, we will have to fetch all remotes + # to be sure... for remote in smmr: remote.fetch(progress=progress) # end for each remote @@ -354,15 +365,16 @@ def update( logmsg="branch: Created from HEAD", ) except OSError: - # ... or reuse the existing one + # ...or reuse the existing one. tbr = git.Head(smm, sm.branch_path) # END assure tracking branch exists tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name)) # NOTE: All head-resetting is done in the base implementation of update - # but we will have to checkout the new branch here. As it still points to the currently - # checkout out commit, we don't do any harm. - # As we don't want to update working-tree or index, changing the ref is all there is to do + # but we will have to checkout the new branch here. As it still points + # to the currently checked out commit, we don't do any harm. + # As we don't want to update working-tree or index, changing the ref is + # all there is to do. smm.head.reference = tbr # END handle dry_run @@ -384,7 +396,7 @@ def update( # FINALLY UPDATE ALL ACTUAL SUBMODULES ###################################### for sm in sms: - # update the submodule using the default method + # Update the submodule using the default method. sm.update( recursive=False, init=init, @@ -395,12 +407,12 @@ def update( keep_going=keep_going, ) - # update recursively depth first - question is which inconsistent + # Update recursively depth first - question is which inconsistent # state will be better in case it fails somewhere. Defective branch # or defective depth. The RootSubmodule type will never process itself, - # which was done in the previous expression + # which was done in the previous expression. if recursive: - # the module would exist by now if we are not in dry_run mode + # The module would exist by now if we are not in dry_run mode. if sm.module_exists(): type(self)(sm.module()).update( recursive=True, @@ -419,7 +431,7 @@ def update( return self def module(self) -> "Repo": - """:return: the actual repository containing the submodules""" + """:return: The actual repository containing the submodules""" return self.repo # } END interface diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 56ce1489a..e13528a8f 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -32,12 +32,12 @@ def sm_section(name: str) -> str: - """:return: section title used in .gitmodules configuration file""" + """:return: Section title used in .gitmodules configuration file""" return f'submodule "{name}"' def sm_name(section: str) -> str: - """:return: name of the submodule as parsed from the section name""" + """:return: Name of the submodule as parsed from the section name""" section = section.strip() return section[11:-1] @@ -48,7 +48,7 @@ def mkhead(repo: "Repo", path: PathLike) -> "Head": def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> "RemoteReference": - """Find the remote branch matching the name of the given branch or raise InvalidGitRepositoryError""" + """Find the remote branch matching the name of the given branch or raise InvalidGitRepositoryError.""" for remote in remotes: try: return remote.refs[branch_name] @@ -66,14 +66,13 @@ def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> " class SubmoduleConfigParser(GitConfigParser): + """Catches calls to _write, and updates the .gitmodules blob in the index + with the new data, if we have written into a stream. - """ - Catches calls to _write, and updates the .gitmodules blob in the index - with the new data, if we have written into a stream. Otherwise it will - add the local file to the index to make it correspond with the working tree. - Additionally, the cache must be cleared + Otherwise it would add the local file to the index to make it correspond + with the working tree. Additionally, the cache must be cleared. - Please note that no mutating method will work in bare mode + Please note that no mutating method will work in bare mode. """ def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -85,13 +84,13 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # { Interface def set_submodule(self, submodule: "Submodule") -> None: """Set this instance's submodule. It must be called before - the first write operation begins""" + the first write operation begins.""" self._smref = weakref.ref(submodule) def flush_to_index(self) -> None: - """Flush changes in our configuration file to the index""" + """Flush changes in our configuration file to the index.""" assert self._smref is not None - # should always have a file here + # Should always have a file here. assert not isinstance(self._file_or_files, BytesIO) sm = self._smref() diff --git a/git/objects/tag.py b/git/objects/tag.py index 56fd05d1a..55f7e19da 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -3,7 +3,9 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ -""" Module containing all object based types. """ + +"""Module containing all Object-based types.""" + from . import base from .util import get_object_type_by_name, parse_actor_and_date from ..util import hex_to_bin @@ -24,10 +26,10 @@ class TagObject(base.Object): - - """Non-Lightweight tag carrying additional information about an object we are pointing to.""" + """Non-lightweight tag carrying additional information about an object we are pointing to.""" type: Literal["tag"] = "tag" + __slots__ = ( "object", "tag", @@ -48,18 +50,20 @@ def __init__( tagger_tz_offset: Union[int, None] = None, message: Union[str, None] = None, ) -> None: # @ReservedAssignment - """Initialize a tag object with additional data + """Initialize a tag object with additional data. - :param repo: repository this object is located in + :param repo: Repository this object is located in :param binsha: 20 byte SHA1 :param object: Object instance of object we are pointing to - :param tag: name of this tag + :param tag: Name of this tag :param tagger: Actor identifying the tagger :param tagged_date: int_seconds_since_epoch - is the DateTime of the tag creation - use time.gmtime to convert - it into a different format - :param tagged_tz_offset: int_seconds_west_of_utc is the timezone that the - authored_date is in, in a format similar to time.altzone""" + The :class:`DateTime` of the tag creation. + Use :func:`time.gmtime` to convert it into a different format. + :param tagged_tz_offset: int_seconds_west_of_utc + The timezone that the authored_date is in, in a format similar + to :attr:`time.altzone`. + """ super(TagObject, self).__init__(repo, binsha) if object is not None: self.object: Union["Commit", "Blob", "Tree", "TagObject"] = object @@ -75,7 +79,7 @@ def __init__( self.message = message def _set_cache_(self, attr: str) -> None: - """Cache all our attributes at once""" + """Cache all our attributes at once.""" if attr in TagObject.__slots__: ostream = self.repo.odb.stream(self.binsha) lines: List[str] = ostream.read().decode(defenc, "replace").splitlines() @@ -95,9 +99,9 @@ def _set_cache_(self, attr: str) -> None: self.tagger_tz_offset, ) = parse_actor_and_date(tagger_info) - # line 4 empty - it could mark the beginning of the next header - # in case there really is no message, it would not exist. Otherwise - # a newline separates header from message + # Line 4 empty - it could mark the beginning of the next header. + # In case there really is no message, it would not exist. + # Otherwise a newline separates header from message. if len(lines) > 5: self.message = "\n".join(lines[5:]) else: diff --git a/git/objects/tree.py b/git/objects/tree.py index 4f490af54..fec98d6e8 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -103,11 +103,11 @@ def merge_sort(a: List[TreeCacheTup], cmp: Callable[[TreeCacheTup, TreeCacheTup] class TreeModifier(object): - """A utility class providing methods to alter the underlying cache in a list-like fashion. Once all adjustments are complete, the _cache, which really is a reference to - the cache of a tree, will be sorted. Assuring it will be in a serializable state""" + the cache of a tree, will be sorted. This ensures it will be in a serializable state. + """ __slots__ = "_cache" @@ -126,10 +126,12 @@ def _index_by_name(self, name: str) -> int: # { Interface def set_done(self) -> "TreeModifier": """Call this method once you are done modifying the tree information. - It may be called several times, but be aware that each call will cause - a sort operation - :return self:""" + This may be called several times, but be aware that each call will cause + a sort operation. + + :return self: + """ merge_sort(self._cache, git_cmp) return self @@ -137,16 +139,21 @@ def set_done(self) -> "TreeModifier": # { Mutators def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeModifier": - """Add the given item to the tree. If an item with the given name already - exists, nothing will be done, but a ValueError will be raised if the - sha and mode of the existing item do not match the one you add, unless - force is True + """Add the given item to the tree. + + If an item with the given name already exists, nothing will be done, but a + ValueError will be raised if the sha and mode of the existing item do not match + the one you add, unless force is True :param sha: The 20 or 40 byte sha of the item to add + :param mode: int representing the stat compatible mode of the item - :param force: If True, an item with your name and information will overwrite - any existing item with the same name, no matter which information it has - :return: self""" + + :param force: If True, an item with your name and information will overwrite any + existing item with the same name, no matter which information it has + + :return: self + """ if "/" in name: raise ValueError("Name must not contain '/' characters") if (mode >> 12) not in Tree._map_id_to_type: @@ -173,18 +180,20 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod return self def add_unchecked(self, binsha: bytes, mode: int, name: str) -> None: - """Add the given item to the tree, its correctness is assumed, which + """Add the given item to the tree. Its correctness is assumed, which puts the caller into responsibility to assure the input is correct. - For more information on the parameters, see ``add`` - :param binsha: 20 byte binary sha""" + For more information on the parameters, see :meth:`add`. + + :param binsha: 20 byte binary sha + """ assert isinstance(binsha, bytes) and isinstance(mode, int) and isinstance(name, str) tree_cache = (binsha, mode, name) self._cache.append(tree_cache) def __delitem__(self, name: str) -> None: - """Deletes an item with the given name if it exists""" + """Delete an item with the given name if it exists.""" index = self._index_by_name(name) if index > -1: del self._cache[index] @@ -193,7 +202,6 @@ def __delitem__(self, name: str) -> None: class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): - """Tree objects represent an ordered list of Blobs and other Trees. ``Tree as a list``:: @@ -208,8 +216,8 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): type: Literal["tree"] = "tree" __slots__ = "_cache" - # actual integer ids for comparison - commit_id = 0o16 # equals stat.S_IFDIR | stat.S_IFLNK - a directory link + # Actual integer IDs for comparison. + commit_id = 0o16 # Equals stat.S_IFDIR | stat.S_IFLNK - a directory link. blob_id = 0o10 symlink_id = 0o12 tree_id = 0o04 @@ -218,7 +226,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): commit_id: Submodule, blob_id: Blob, symlink_id: Blob - # tree id added once Tree is defined + # Tree ID added once Tree is defined. } def __init__( @@ -241,7 +249,7 @@ def _get_intermediate_items( def _set_cache_(self, attr: str) -> None: if attr == "_cache": - # Set the data when we need it + # Set the data when we need it. ostream = self.repo.odb.stream(self.binsha) self._cache: List[TreeCacheTup] = tree_entries_from_data(ostream.read()) else: @@ -250,7 +258,8 @@ def _set_cache_(self, attr: str) -> None: def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[IndexObjUnion]: """Iterable yields tuples of (binsha, mode, name), which will be converted - to the respective object representation""" + to the respective object representation. + """ for binsha, mode, name in iterable: path = join_path(self.path, name) try: @@ -260,10 +269,11 @@ def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[ # END for each item def join(self, file: str) -> IndexObjUnion: - """Find the named object in this tree's contents + """Find the named object in this tree's contents. :return: ``git.Blob`` or ``git.Tree`` or ``git.Submodule`` - :raise KeyError: if given file or tree does not exist in tree""" + :raise KeyError: if given file or tree does not exist in tree + """ msg = "Blob or Tree named %r not found" if "/" in file: tree = self @@ -274,7 +284,7 @@ def join(self, file: str) -> IndexObjUnion: if item.type == "tree": tree = item else: - # safety assertion - blobs are at the end of the path + # Safety assertion - blobs are at the end of the path. if i != len(tokens) - 1: raise KeyError(msg % file) return item @@ -294,7 +304,10 @@ def join(self, file: str) -> IndexObjUnion: # END handle long paths def __truediv__(self, file: str) -> IndexObjUnion: - """For PY3 only""" + """The ``/`` operator is another syntax for joining. + + See :meth:`join` for details. + """ return self.join(file) @property @@ -313,7 +326,8 @@ def cache(self) -> TreeModifier: :return: An object allowing to modify the internal cache. This can be used to change the tree's contents. When done, make sure you call ``set_done`` on the tree modifier, or serialization behaviour will be incorrect. - See the ``TreeModifier`` for more information on how to alter the cache""" + See :class:`TreeModifier` for more information on how to alter the cache. + """ return TreeModifier(self._cache) def traverse( @@ -326,8 +340,10 @@ def traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]]: - """For documentation, see util.Traversable._traverse() - Trees are set to visit_once = False to gain more performance in the traversal""" + """For documentation, see util.Traversable._traverse(). + + Trees are set to ``visit_once = False`` to gain more performance in the traversal. + """ # """ # # To typecheck instead of using cast. @@ -392,7 +408,7 @@ def __contains__(self, item: Union[IndexObjUnion, PathLike]) -> bool: # END handle item is index object # compatibility - # treat item as repo-relative path + # Treat item as repo-relative path. else: path = self.path for info in self._cache: @@ -405,10 +421,12 @@ def __reversed__(self) -> Iterator[IndexObjUnion]: return reversed(self._iter_convert_to_object(self._cache)) # type: ignore def _serialize(self, stream: "BytesIO") -> "Tree": - """Serialize this tree into the stream. Please note that we will assume - our tree data to be in a sorted state. If this is not the case, serialization - will not generate a correct tree representation as these are assumed to be sorted - by algorithms""" + """Serialize this tree into the stream. Assumes sorted tree data. + + .. note:: We will assume our tree data to be in a sorted state. If this is not + the case, serialization will not generate a correct tree representation as + these are assumed to be sorted by algorithms. + """ tree_to_stream(self._cache, stream.write) return self @@ -419,6 +437,5 @@ def _deserialize(self, stream: "BytesIO") -> "Tree": # END tree -# finalize map definition +# Finalize map definition. Tree._map_id_to_type[Tree.tree_id] = Tree -# diff --git a/git/objects/util.py b/git/objects/util.py index 992a53d9c..d2c1c0158 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -3,7 +3,9 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module for general utility functions""" + +"""Module for general utility functions.""" + # flake8: noqa F401 @@ -62,10 +64,10 @@ class TraverseNT(NamedTuple): src: Union["Traversable", None] -T_TIobj = TypeVar("T_TIobj", bound="TraversableIterableObj") # for TraversableIterableObj.traverse() +T_TIobj = TypeVar("T_TIobj", bound="TraversableIterableObj") # For TraversableIterableObj.traverse() TraversedTup = Union[ - Tuple[Union["Traversable", None], "Traversable"], # for commit, submodule + Tuple[Union["Traversable", None], "Traversable"], # For commit, submodule "TraversedTreeTup", ] # for tree.traverse() @@ -92,12 +94,14 @@ class TraverseNT(NamedTuple): def mode_str_to_int(modestr: Union[bytes, str]) -> int: """ - :param modestr: string like 755 or 644 or 100644 - only the last 6 chars will be used + :param modestr: + String like 755 or 644 or 100644 - only the last 6 chars will be used. + :return: String identifying a mode compatible to the mode methods ids of the stat module regarding the rwx permissions for user, group and other, - special flags and file system flags, i.e. whether it is a symlink - for example.""" + special flags and file system flags, such as whether it is a symlink. + """ mode = 0 for iteration, char in enumerate(reversed(modestr[-6:])): char = cast(Union[str, int], char) @@ -110,12 +114,13 @@ def get_object_type_by_name( object_type_name: bytes, ) -> Union[Type["Commit"], Type["TagObject"], Type["Tree"], Type["Blob"]]: """ - :return: type suitable to handle the given object type name. + :return: A type suitable to handle the given object type name. Use the type to create new instances. :param object_type_name: Member of TYPES - :raise ValueError: In case object_type_name is unknown""" + :raise ValueError: If object_type_name is unknown + """ if object_type_name == b"commit": from . import commit @@ -138,9 +143,9 @@ def get_object_type_by_name( def utctz_to_altz(utctz: str) -> int: """Convert a git timezone offset into a timezone offset west of - UTC in seconds (compatible with time.altzone). + UTC in seconds (compatible with :attr:`time.altzone`). - :param utctz: git utc timezone string, i.e. +0200 + :param utctz: git utc timezone string, e.g. +0200 """ int_utctz = int(utctz) seconds = (abs(int_utctz) // 100) * 3600 + (abs(int_utctz) % 100) * 60 @@ -148,9 +153,9 @@ def utctz_to_altz(utctz: str) -> int: def altz_to_utctz_str(altz: float) -> str: - """Convert a timezone offset west of UTC in seconds into a git timezone offset string + """Convert a timezone offset west of UTC in seconds into a Git timezone offset string. - :param altz: timezone offset in seconds west of UTC + :param altz: Timezone offset in seconds west of UTC """ hours = abs(altz) // 3600 minutes = (abs(altz) % 3600) // 60 @@ -159,8 +164,11 @@ def altz_to_utctz_str(altz: float) -> str: def verify_utctz(offset: str) -> str: - """:raise ValueError: if offset is incorrect - :return: offset""" + """ + :raise ValueError: If offset is incorrect + + :return: offset + """ fmt_exc = ValueError("Invalid timezone offset format: %s" % offset) if len(offset) != 5: raise fmt_exc @@ -194,7 +202,7 @@ def dst(self, dt: Union[datetime, None]) -> timedelta: def from_timestamp(timestamp: float, tz_offset: float) -> datetime: - """Converts a timestamp + tz_offset into an aware datetime instance.""" + """Convert a timestamp + tz_offset into an aware datetime instance.""" utc_dt = datetime.fromtimestamp(timestamp, utc) try: local_dt = utc_dt.astimezone(tzoffset(tz_offset)) @@ -205,16 +213,18 @@ def from_timestamp(timestamp: float, tz_offset: float) -> datetime: def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: """ - Parse the given date as one of the following + Parse the given date as one of the following: - * aware datetime instance + * Aware datetime instance * Git internal format: timestamp offset * RFC 2822: Thu, 07 Apr 2005 22:13:13 +0200. * ISO 8601 2005-04-07T22:13:13 - The T can be a space as well + The T can be a space as well. :return: Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch + :raise ValueError: If the format could not be understood + :note: Date can also be YYYY.MM.DD, MM/DD/YYYY and DD.MM.YYYY. """ if isinstance(string_date, datetime): @@ -225,7 +235,7 @@ def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: else: raise ValueError(f"string_date datetime object without tzinfo, {string_date}") - # git time + # Git time try: if string_date.count(" ") == 1 and string_date.rfind(":") == -1: timestamp, offset_str = string_date.split() @@ -234,21 +244,21 @@ def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: timestamp_int = int(timestamp) return timestamp_int, utctz_to_altz(verify_utctz(offset_str)) else: - offset_str = "+0000" # local time by default + offset_str = "+0000" # Local time by default. if string_date[-5] in "-+": offset_str = verify_utctz(string_date[-5:]) string_date = string_date[:-6] # skip space as well # END split timezone info offset = utctz_to_altz(offset_str) - # now figure out the date and time portion - split time + # Now figure out the date and time portion - split time. date_formats = [] splitter = -1 if "," in string_date: date_formats.append("%a, %d %b %Y") splitter = string_date.rfind(" ") else: - # iso plus additional + # ISO plus additional date_formats.append("%Y-%m-%d") date_formats.append("%Y.%m.%d") date_formats.append("%m/%d/%Y") @@ -258,15 +268,15 @@ def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: if splitter == -1: splitter = string_date.rfind(" ") # END handle 'T' and ' ' - # END handle rfc or iso + # END handle RFC or ISO assert splitter > -1 - # split date and time - time_part = string_date[splitter + 1 :] # skip space + # Split date and time. + time_part = string_date[splitter + 1 :] # Skip space. date_part = string_date[:splitter] - # parse time + # Parse time. tstruct = time.strptime(time_part, "%H:%M:%S") for fmt in date_formats: @@ -291,7 +301,7 @@ def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: # END exception handling # END for each fmt - # still here ? fail + # Still here ? fail. raise ValueError("no format matched") # END handle format except Exception as e: @@ -299,7 +309,7 @@ def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: # END handle exceptions -# precompiled regex +# Precompiled regexes _re_actor_epoch = re.compile(r"^.+? (.*) (\d+) ([+-]\d+).*$") _re_only_actor = re.compile(r"^.+? (.*)$") @@ -309,7 +319,8 @@ def parse_actor_and_date(line: str) -> Tuple[Actor, int, int]: author Tom Preston-Werner 1191999972 -0700 - :return: [Actor, int_seconds_since_epoch, int_timezone_offset]""" + :return: [Actor, int_seconds_since_epoch, int_timezone_offset] + """ actor, epoch, offset = "", "0", "0" m = _re_actor_epoch.search(line) if m: @@ -327,12 +338,12 @@ def parse_actor_and_date(line: str) -> Tuple[Actor, int, int]: class ProcessStreamAdapter(object): - - """Class wireing all calls to the contained Process instance. + """Class wiring all calls to the contained Process instance. Use this type to hide the underlying process to provide access only to a specified stream. The process is usually wrapped into an AutoInterrupt class to kill - it if the instance goes out of scope.""" + it if the instance goes out of scope. + """ __slots__ = ("_proc", "_stream") @@ -346,11 +357,12 @@ def __getattr__(self, attr: str) -> Any: @runtime_checkable class Traversable(Protocol): - """Simple interface to perform depth-first or breadth-first traversals - into one direction. + in one direction. + Subclasses only need to implement one function. - Instances of the Subclass must be hashable + + Instances of the Subclass must be hashable. Defined subclasses = [Commit, Tree, SubModule] """ @@ -363,7 +375,7 @@ def _get_intermediate_items(cls, item: Any) -> Sequence["Traversable"]: """ Returns: Tuple of items connected to the given item. - Must be implemented in subclass + Must be implemented in subclass. class Commit:: (cls, Commit) -> Tuple[Commit, ...] class Submodule:: (cls, Submodule) -> Iterablelist[Submodule] @@ -393,22 +405,22 @@ def _list_traverse( Submodule -> IterableList['Submodule'] Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] """ - # Commit and Submodule have id.__attribute__ as IterableObj - # Tree has id.__attribute__ inherited from IndexObject + # Commit and Submodule have id.__attribute__ as IterableObj. + # Tree has id.__attribute__ inherited from IndexObject. if isinstance(self, Has_id_attribute): id = self._id_attribute_ else: - id = "" # shouldn't reach here, unless Traversable subclass created with no _id_attribute_ - # could add _id_attribute_ to Traversable, or make all Traversable also Iterable? + id = "" # Shouldn't reach here, unless Traversable subclass created with no _id_attribute_. + # Could add _id_attribute_ to Traversable, or make all Traversable also Iterable? if not as_edge: out: IterableList[Union["Commit", "Submodule", "Tree", "Blob"]] = IterableList(id) out.extend(self.traverse(as_edge=as_edge, *args, **kwargs)) return out - # overloads in subclasses (mypy doesn't allow typing self: subclass) + # Overloads in subclasses (mypy doesn't allow typing self: subclass). # Union[IterableList['Commit'], IterableList['Submodule'], IterableList[Union['Submodule', 'Tree', 'Blob']]] else: - # Raise deprecationwarning, doesn't make sense to use this + # Raise DeprecationWarning, it doesn't make sense to use this. out_list: IterableList = IterableList(self.traverse(*args, **kwargs)) return out_list @@ -434,35 +446,37 @@ def _traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[Union["Traversable", "Blob"]], Iterator[TraversedTup]]: - """:return: iterator yielding of items found when traversing self - :param predicate: f(i,d) returns False if item i at depth d should not be included in the result + """:return: Iterator yielding items found when traversing self + + :param predicate: f(i,d) returns False if item i at depth d should not be + included in the result. :param prune: - f(i,d) return True if the search should stop at item i at depth d. - Item i will not be returned. + f(i,d) return True if the search should stop at item i at depth d. Item i + will not be returned. :param depth: - define at which level the iteration should not go deeper - if -1, there is no limit - if 0, you would effectively only get self, the root of the iteration + Defines at which level the iteration should not go deeper if -1, there is no + limit if 0, you would effectively only get self, the root of the iteration i.e. if 1, you would only get the first level of predecessors/successors :param branch_first: if True, items will be returned branch first, otherwise depth first :param visit_once: - if True, items will only be returned once, although they might be encountered - several times. Loops are prevented that way. + if True, items will only be returned once, although they might be + encountered several times. Loops are prevented that way. :param ignore_self: - if True, self will be ignored and automatically pruned from - the result. Otherwise it will be the first item to be returned. - If as_edge is True, the source of the first edge is None + if True, self will be ignored and automatically pruned from the result. + Otherwise it will be the first item to be returned. If as_edge is True, the + source of the first edge is None :param as_edge: if True, return a pair of items, first being the source, second the - destination, i.e. tuple(src, dest) with the edge spanning from - source to destination""" + destination, i.e. tuple(src, dest) with the edge spanning from source to + destination + """ """ Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]] @@ -473,11 +487,12 @@ def _traverse( ignore_self=True is_edge=True -> Iterator[item] ignore_self=True is_edge=False --> Iterator[item] ignore_self=False is_edge=True -> Iterator[item] | Iterator[Tuple[src, item]] - ignore_self=False is_edge=False -> Iterator[Tuple[src, item]]""" + ignore_self=False is_edge=False -> Iterator[Tuple[src, item]] + """ visited = set() stack: Deque[TraverseNT] = deque() - stack.append(TraverseNT(0, self, None)) # self is always depth level 0 + stack.append(TraverseNT(0, self, None)) # self is always depth level 0. def addToStack( stack: Deque[TraverseNT], @@ -497,7 +512,7 @@ def addToStack( # END addToStack local method while stack: - d, item, src = stack.pop() # depth of item, item, item_source + d, item, src = stack.pop() # Depth of item, item, item_source if visit_once and item in visited: continue @@ -506,7 +521,7 @@ def addToStack( visited.add(item) rval: Union[TraversedTup, "Traversable", "Blob"] - if as_edge: # if as_edge return (src, item) unless rrc is None (e.g. for first item) + if as_edge: # If as_edge return (src, item) unless rrc is None (e.g. for first item). rval = (src, item) else: rval = item @@ -518,7 +533,7 @@ def addToStack( if not skipStartItem and predicate(rval, d): yield rval - # only continue to next level if this is appropriate ! + # Only continue to next level if this is appropriate! nd = d + 1 if depth > -1 and nd > depth: continue @@ -529,24 +544,30 @@ def addToStack( @runtime_checkable class Serializable(Protocol): - - """Defines methods to serialize and deserialize objects from and into a data stream""" + """Defines methods to serialize and deserialize objects from and into a data stream.""" __slots__ = () # @abstractmethod def _serialize(self, stream: "BytesIO") -> "Serializable": - """Serialize the data of this object into the given data stream - :note: a serialized object would ``_deserialize`` into the same object + """Serialize the data of this object into the given data stream. + + :note: A serialized object would ``_deserialize`` into the same object. + :param stream: a file-like object - :return: self""" + + :return: self + """ raise NotImplementedError("To be implemented in subclass") # @abstractmethod def _deserialize(self, stream: "BytesIO") -> "Serializable": - """Deserialize all information regarding this object from the stream + """Deserialize all information regarding this object from the stream. + :param stream: a file-like object - :return: self""" + + :return: self + """ raise NotImplementedError("To be implemented in subclass") diff --git a/git/refs/__init__.py b/git/refs/__init__.py index 1486dffe6..18ea2013c 100644 --- a/git/refs/__init__.py +++ b/git/refs/__init__.py @@ -1,5 +1,5 @@ # flake8: noqa -# import all modules in order, fix the names they require +# Import all modules in order, fix the names they require. from .symbolic import * from .reference import * from .head import * diff --git a/git/refs/head.py b/git/refs/head.py index 26efc6cb9..194f51e78 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -5,7 +5,7 @@ from .symbolic import SymbolicReference from .reference import Reference -# typinng --------------------------------------------------- +# typing --------------------------------------------------- from typing import Any, Sequence, Union, TYPE_CHECKING @@ -28,12 +28,12 @@ def strip_quotes(string: str) -> str: class HEAD(SymbolicReference): - - """Special case of a Symbolic Reference as it represents the repository's + """Special case of a SymbolicReference representing the repository's HEAD reference.""" _HEAD_NAME = "HEAD" _ORIG_HEAD_NAME = "ORIG_HEAD" + __slots__ = () def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): @@ -45,7 +45,8 @@ def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): def orig_head(self) -> SymbolicReference: """ :return: SymbolicReference pointing at the ORIG_HEAD, which is maintained - to contain the previous value of HEAD""" + to contain the previous value of HEAD. + """ return SymbolicReference(self.repo, self._ORIG_HEAD_NAME) def reset( @@ -71,7 +72,7 @@ def reset( :param working_tree: If True, the working tree will be forcefully adjusted to match the given commit, possibly overwriting uncommitted changes without warning. - If working_tree is True, index must be true as well + If `working_tree` is True, `index` must be True as well. :param paths: Single path or list of paths relative to the git root directory @@ -80,14 +81,15 @@ def reset( :param kwargs: Additional arguments passed to git-reset. - :return: self""" + :return: self + """ mode: Union[str, None] mode = "--soft" if index: mode = "--mixed" - # it appears, some git-versions declare mixed and paths deprecated - # see http://github.com/Byron/GitPython/issues#issue/2 + # Tt appears some git versions declare mixed and paths deprecated. + # See http://github.com/Byron/GitPython/issues#issue/2. if paths: mode = None # END special case @@ -104,7 +106,7 @@ def reset( self.repo.git.reset(mode, commit, "--", paths, **kwargs) except GitCommandError as e: # git nowadays may use 1 as status to indicate there are still unstaged - # modifications after the reset + # modifications after the reset. if e.status != 1: raise # END handle exception @@ -113,7 +115,6 @@ def reset( class Head(Reference): - """A Head is a named reference to a Commit. Every Head instance contains a name and a Commit object. @@ -129,33 +130,35 @@ class Head(Reference): >>> head.commit.hexsha - '1c09f116cbc2cb4100fb6935bb162daa4723f455'""" + '1c09f116cbc2cb4100fb6935bb162daa4723f455' + """ _common_path_default = "refs/heads" k_config_remote = "remote" - k_config_remote_ref = "merge" # branch to merge from remote + k_config_remote_ref = "merge" # Branch to merge from remote. @classmethod def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: - """Delete the given heads + """Delete the given heads. :param force: If True, the heads will be deleted even if they are not yet merged into the main development stream. - Default False""" + Default False + """ flag = "-d" if force: flag = "-D" repo.git.branch(flag, *heads) def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) -> "Head": - """ - Configure this branch to track the given remote reference. This will alter - this branch's configuration accordingly. + """Configure this branch to track the given remote reference. This will + alter this branch's configuration accordingly. :param remote_reference: The remote reference to track or None to untrack - any references - :return: self""" + any references. + :return: self + """ from .remote import RemoteReference if remote_reference is not None and not isinstance(remote_reference, RemoteReference): @@ -180,7 +183,7 @@ def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) def tracking_branch(self) -> Union["RemoteReference", None]: """ :return: The remote_reference we are tracking, or None if we are - not a tracking branch""" + not a tracking branch.""" from .remote import RemoteReference reader = self.config_reader() @@ -193,22 +196,24 @@ def tracking_branch(self) -> Union["RemoteReference", None]: return RemoteReference(self.repo, remote_refpath) # END handle have tracking branch - # we are not a tracking branch + # We are not a tracking branch. return None def rename(self, new_path: PathLike, force: bool = False) -> "Head": - """Rename self to a new path + """Rename self to a new path. :param new_path: Either a simple name or a path, i.e. new_name or features/new_name. - The prefix refs/heads is implied + The prefix refs/heads is implied. :param force: If True, the rename will succeed even if a head with the target name already exists. :return: self - :note: respects the ref log as git commands are used""" + + :note: Respects the ref log as git commands are used. + """ flag = "-m" if force: flag = "-M" @@ -218,19 +223,20 @@ def rename(self, new_path: PathLike, force: bool = False) -> "Head": return self def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: - """Checkout this head by setting the HEAD to this reference, by updating the index - to reflect the tree we point to and by updating the working tree to reflect - the latest index. + """Check out this head by setting the HEAD to this reference, by updating the + index to reflect the tree we point to and by updating the working tree to + reflect the latest index. The command will fail if changed working tree files would be overwritten. :param force: If True, changes to the index and the working tree will be discarded. - If False, GitCommandError will be raised in that situation. + If False, :class:`GitCommandError ` will be + raised in that situation. :param kwargs: - Additional keyword arguments to be passed to git checkout, i.e. - b='new_branch' to create a new branch at the given spot. + Additional keyword arguments to be passed to git checkout, e.g. + ``b="new_branch"`` to create a new branch at the given spot. :return: The active branch after the checkout operation, usually self unless @@ -241,7 +247,8 @@ def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: :note: By default it is only allowed to checkout heads - everything else will leave the HEAD detached which is allowed and possible, but remains - a special state that some tools might not be able to handle.""" + a special state that some tools might not be able to handle. + """ kwargs["f"] = force if kwargs["f"] is False: kwargs.pop("f") @@ -265,13 +272,15 @@ def _config_parser(self, read_only: bool) -> SectionConstraint[GitConfigParser]: def config_reader(self) -> SectionConstraint[GitConfigParser]: """ :return: A configuration parser instance constrained to only read - this instance's values""" + this instance's values. + """ return self._config_parser(read_only=True) def config_writer(self) -> SectionConstraint[GitConfigParser]: """ :return: A configuration writer instance with read-and write access - to options of this head""" + to options of this head. + """ return self._config_parser(read_only=False) # } END configuration diff --git a/git/refs/log.py b/git/refs/log.py index 9b02051d3..21c757ccd 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -38,18 +38,17 @@ class RefLogEntry(Tuple[str, str, Actor, Tuple[int, int], str]): - - """Named tuple allowing easy access to the revlog data fields""" + """Named tuple allowing easy access to the revlog data fields.""" _re_hexsha_only = re.compile(r"^[0-9A-Fa-f]{40}$") __slots__ = () def __repr__(self) -> str: - """Representation of ourselves in git reflog format""" + """Representation of ourselves in git reflog format.""" return self.format() def format(self) -> str: - """:return: a string suitable to be placed in a reflog file""" + """:return: A string suitable to be placed in a reflog file.""" act = self.actor time = self.time return "{} {} {} <{}> {!s} {}\t{}\n".format( @@ -64,30 +63,31 @@ def format(self) -> str: @property def oldhexsha(self) -> str: - """The hexsha to the commit the ref pointed to before the change""" + """The hexsha to the commit the ref pointed to before the change.""" return self[0] @property def newhexsha(self) -> str: - """The hexsha to the commit the ref now points to, after the change""" + """The hexsha to the commit the ref now points to, after the change.""" return self[1] @property def actor(self) -> Actor: - """Actor instance, providing access""" + """Actor instance, providing access.""" return self[2] @property def time(self) -> Tuple[int, int]: """time as tuple: - * [0] = int(time) - * [1] = int(timezone_offset) in time.altzone format""" + * [0] = ``int(time)`` + * [1] = ``int(timezone_offset)`` in :attr:`time.altzone` format + """ return self[3] @property def message(self) -> str: - """Message describing the operation that acted on the reference""" + """Message describing the operation that acted on the reference.""" return self[4] @classmethod @@ -109,8 +109,11 @@ def new( @classmethod def from_line(cls, line: bytes) -> "RefLogEntry": """:return: New RefLogEntry instance from the given revlog line. - :param line: line bytes without trailing newline - :raise ValueError: If line could not be parsed""" + + :param line: Line bytes without trailing newline + + :raise ValueError: If `line` could not be parsed + """ line_str = line.decode(defenc) fields = line_str.split("\t", 1) if len(fields) == 1: @@ -141,13 +144,13 @@ def from_line(cls, line: bytes) -> "RefLogEntry": class RefLog(List[RefLogEntry], Serializable): - """A reflog contains RefLogEntrys, each of which defines a certain state of the head in question. Custom query methods allow to retrieve log entries by date or by other criteria. - Reflog entries are ordered, the first added entry is first in the list, the last - entry, i.e. the last change of the head or reference, is last in the list.""" + Reflog entries are ordered. The first added entry is first in the list. The last + entry, i.e. the last change of the head or reference, is last in the list. + """ __slots__ = ("_path",) @@ -158,7 +161,7 @@ def __new__(cls, filepath: Union[PathLike, None] = None) -> "RefLog": def __init__(self, filepath: Union[PathLike, None] = None): """Initialize this instance with an optional filepath, from which we will initialize our data. The path is also used to write changes back using - the write() method""" + the write() method.""" self._path = filepath if filepath is not None: self._read_from_file() @@ -168,7 +171,7 @@ def _read_from_file(self) -> None: try: fmap = file_contents_ro_filepath(self._path, stream=True, allow_mmap=True) except OSError: - # it is possible and allowed that the file doesn't exist ! + # It is possible and allowed that the file doesn't exist! return # END handle invalid log @@ -183,31 +186,35 @@ def _read_from_file(self) -> None: @classmethod def from_file(cls, filepath: PathLike) -> "RefLog": """ - :return: a new RefLog instance containing all entries from the reflog + :return: A new RefLog instance containing all entries from the reflog at the given filepath - :param filepath: path to reflog - :raise ValueError: If the file could not be read or was corrupted in some way""" + :param filepath: Path to reflog + :raise ValueError: If the file could not be read or was corrupted in some way + """ return cls(filepath) @classmethod def path(cls, ref: "SymbolicReference") -> str: """ - :return: string to absolute path at which the reflog of the given ref + :return: String to absolute path at which the reflog of the given ref instance would be found. The path is not guaranteed to point to a valid file though. - :param ref: SymbolicReference instance""" + :param ref: SymbolicReference instance + """ return osp.join(ref.repo.git_dir, "logs", to_native_path(ref.path)) @classmethod def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEntry]: """ :return: Iterator yielding RefLogEntry instances, one for each line read - sfrom the given stream. - :param stream: file-like object containing the revlog in its native format - or string instance pointing to a file to read""" + from the given stream. + + :param stream: File-like object containing the revlog in its native format + or string instance pointing to a file to read. + """ new_entry = RefLogEntry.from_line if isinstance(stream, str): - # default args return mmap on py>3 + # Default args return mmap since Python 3. _stream = file_contents_ro_filepath(stream) assert isinstance(_stream, mmap) else: @@ -223,23 +230,23 @@ def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEnt @classmethod def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": """ - :return: RefLogEntry at the given index + :return: RefLogEntry at the given index. - :param filepath: full path to the index file from which to read the entry + :param filepath: Full path to the index file from which to read the entry. - :param index: python list compatible index, i.e. it may be negative to - specify an entry counted from the end of the list + :param index: Python list compatible index, i.e. it may be negative to + specify an entry counted from the end of the list. - :raise IndexError: If the entry didn't exist + :raise IndexError: If the entry didn't exist. .. note:: This method is faster as it only parses the entry at index, skipping all other lines. Nonetheless, the whole file has to be read if - the index is negative + the index is negative. """ with open(filepath, "rb") as fp: if index < 0: return RefLogEntry.from_line(fp.readlines()[index].strip()) - # read until index is reached + # Read until index is reached. for i in range(index + 1): line = fp.readline() @@ -254,7 +261,8 @@ def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": def to_file(self, filepath: PathLike) -> None: """Write the contents of the reflog instance to a file at the given filepath. - :param filepath: path to file, parent directories are assumed to exist""" + :param filepath: Path to file, parent directories are assumed to exist. + """ lfd = LockedFD(filepath) assure_directory_exists(filepath, is_file=True) @@ -279,19 +287,21 @@ def append_entry( ) -> "RefLogEntry": """Append a new log entry to the revlog at filepath. - :param config_reader: configuration reader of the repository - used to obtain - user information. May also be an Actor instance identifying the committer directly or None. - :param filepath: full path to the log file - :param oldbinsha: binary sha of the previous commit - :param newbinsha: binary sha of the current commit - :param message: message describing the change to the reference - :param write: If True, the changes will be written right away. Otherwise - the change will not be written + :param config_reader: Configuration reader of the repository - used to obtain + user information. May also be an Actor instance identifying the committer + directly or None. + :param filepath: Full path to the log file. + :param oldbinsha: Binary sha of the previous commit. + :param newbinsha: Binary sha of the current commit. + :param message: Message describing the change to the reference. + :param write: If True, the changes will be written right away. Otherwise the + change will not be written. - :return: RefLogEntry objects which was appended to the log + :return: RefLogEntry objects which was appended to the log. - :note: As we are append-only, concurrent access is not a problem as we - do not interfere with readers.""" + :note: As we are append-only, concurrent access is not a problem as we do not + interfere with readers. + """ if len(oldbinsha) != 20 or len(newbinsha) != 20: raise ValueError("Shas need to be given in binary format") @@ -325,9 +335,10 @@ def append_entry( return entry def write(self) -> "RefLog": - """Write this instance's data to the file we are originating from + """Write this instance's data to the file we are originating from. - :return: self""" + :return: self + """ if self._path is None: raise ValueError("Instance was not initialized with a path, use to_file(...) instead") # END assert path @@ -337,10 +348,11 @@ def write(self) -> "RefLog": # } END interface # { Serializable Interface + def _serialize(self, stream: "BytesIO") -> "RefLog": write = stream.write - # write all entries + # Write all entries. for e in self: write(e.format().encode(defenc)) # END for each entry @@ -348,5 +360,6 @@ def _serialize(self, stream: "BytesIO") -> "RefLog": def _deserialize(self, stream: "BytesIO") -> "RefLog": self.extend(self.iter_entries(stream)) - # } END serializable interface return self + + # } END serializable interface diff --git a/git/refs/reference.py b/git/refs/reference.py index dea1af68c..ca265c0e4 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -22,7 +22,7 @@ def require_remote_ref_path(func: Callable[..., _T]) -> Callable[..., _T]: - """A decorator raising a TypeError if we are not a valid remote, based on the path""" + """A decorator raising a TypeError if we are not a valid remote, based on the path.""" def wrapper(self: T_References, *args: Any) -> _T: if not self.is_remote(): @@ -38,27 +38,30 @@ def wrapper(self: T_References, *args: Any) -> _T: class Reference(SymbolicReference, LazyMixin, IterableObj): + """A named reference to any object. - """Represents a named reference to any object. Subclasses may apply restrictions though, - i.e. Heads can only point to commits.""" + Subclasses may apply restrictions though, e.g., a :class:`Head ` + can only point to commits. + """ __slots__ = () + _points_to_commits_only = False _resolve_ref_on_create = True _common_path_default = "refs" def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> None: - """Initialize this instance - - :param repo: Our parent repository - :param path: - Path relative to the .git/ directory pointing to the ref in question, i.e. - refs/heads/master - :param check_path: if False, you can provide any path. Otherwise the path must start with the - default path prefix of this type.""" + """Initialize this instance. + + :param repo: Our parent repository. + :param path: Path relative to the .git/ directory pointing to the ref in + question, e.g. ``refs/heads/master``. + :param check_path: If False, you can provide any path. Otherwise the path must + start with the default path prefix of this type. + """ if check_path and not str(path).startswith(self._common_path_default + "/"): raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}") - self.path: str # SymbolicReference converts to string atm + self.path: str # SymbolicReference converts to string at the moment. super(Reference, self).__init__(repo, path) def __str__(self) -> str: @@ -72,9 +75,10 @@ def set_object( object: Union[Commit_ish, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "Reference": - """Special version which checks if the head-log needs an update as well + """Special version which checks if the head-log needs an update as well. - :return: self""" + :return: self + """ oldbinsha = None if logmsg is not None: head = self.repo.head @@ -104,13 +108,13 @@ def set_object( return self - # NOTE: Don't have to overwrite properties as the will only work without a the log + # NOTE: No need to overwrite properties, as the will only work without a the log. @property def name(self) -> str: """:return: (shortest) Name of this reference - it may contain path components""" - # first two path tokens are can be removed as they are - # refs/heads or refs/tags or refs/remotes + # The first two path tokens can be removed as they are + # refs/heads or refs/tags or refs/remotes. tokens = self.path.split("/") if len(tokens) < 3: return self.path # could be refs/HEAD @@ -132,23 +136,27 @@ def iter_items( # { Remote Interface - @property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21) + @property # type: ignore # mypy cannot deal with properties with an extra decorator (2021-04-21). @require_remote_ref_path def remote_name(self) -> str: """ :return: Name of the remote we are a reference of, such as 'origin' for a reference - named 'origin/master'""" + named 'origin/master'. + """ tokens = self.path.split("/") # /refs/remotes// return tokens[2] - @property # type: ignore ## mypy cannot deal with properties with an extra decorator (2021-04-21) + @property # type: ignore # mypy cannot deal with properties with an extra decorator (2021-04-21). @require_remote_ref_path def remote_head(self) -> str: - """:return: Name of the remote head itself, i.e. master. + """ + :return: Name of the remote head itself, e.g. master. + :note: The returned name is usually not qualified enough to uniquely identify - a branch""" + a branch. + """ tokens = self.path.split("/") return "/".join(tokens[3:]) diff --git a/git/refs/remote.py b/git/refs/remote.py index ec10c5a1b..e4b1f4392 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -21,8 +21,7 @@ class RemoteReference(Head): - - """Represents a reference pointing to a remote head.""" + """A reference pointing to a remote head.""" _common_path_default = Head._remote_common_path_default @@ -35,7 +34,7 @@ def iter_items( *args: Any, **kwargs: Any, ) -> Iterator["RemoteReference"]: - """Iterate remote references, and if given, constrain them to the given remote""" + """Iterate remote references, and if given, constrain them to the given remote.""" common_path = common_path or cls._common_path_default if remote is not None: common_path = join_path(common_path, str(remote)) @@ -49,15 +48,16 @@ def iter_items( # "type: ignore". (See https://github.com/python/typing/issues/241) @classmethod def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore - """Delete the given remote references + """Delete the given remote references. :note: kwargs are given for comparability with the base class method as we - should not narrow the signature.""" + should not narrow the signature. + """ repo.git.branch("-d", "-r", *refs) - # the official deletion method will ignore remote symbolic refs - these + # The official deletion method will ignore remote symbolic refs - these # are generally ignored in the refs/ folder. We don't though - # and delete remainders manually + # and delete remainders manually. for ref in refs: try: os.remove(os.path.join(repo.common_dir, ref.path)) @@ -71,5 +71,5 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: @classmethod def create(cls, *args: Any, **kwargs: Any) -> NoReturn: - """Used to disable this method""" + """Raises TypeError. Defined so the create method is disabled.""" raise TypeError("Cannot explicitly create remote references") diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 5ff84367d..7da957e2f 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -48,7 +48,7 @@ def _git_dir(repo: "Repo", path: Union[PathLike, None]) -> PathLike: - """Find the git dir that's appropriate for the path""" + """Find the git dir that is appropriate for the path.""" name = f"{path}" if name in ["HEAD", "ORIG_HEAD", "FETCH_HEAD", "index", "logs"]: return repo.git_dir @@ -56,14 +56,16 @@ def _git_dir(repo: "Repo", path: Union[PathLike, None]) -> PathLike: class SymbolicReference(object): + """Special case of a reference that is symbolic. - """Represents a special case of a reference such that this reference is symbolic. - It does not point to a specific commit, but to another Head, which itself - specifies a commit. + This does not point to a specific commit, but to another + :class:`Head `, which itself specifies a commit. - A typical example for a symbolic reference is HEAD.""" + A typical example for a symbolic reference is ``HEAD``. + """ __slots__ = ("repo", "path") + _resolve_ref_on_create = False _points_to_commits_only = True _common_path_default = "" @@ -97,7 +99,8 @@ def name(self) -> str: """ :return: In case of symbolic references, the shortest assumable name - is the path itself.""" + is the path itself. + """ return str(self.path) @property @@ -110,8 +113,11 @@ def _get_packed_refs_path(cls, repo: "Repo") -> str: @classmethod def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: - """Returns an iterator yielding pairs of sha1/path pairs (as strings) for the corresponding refs. - :note: The packed refs file will be kept open as long as we iterate""" + """Return an iterator yielding pairs of sha1/path pairs (as strings) + for the corresponding refs. + + :note: The packed refs file will be kept open as long as we iterate. + """ try: with open(cls._get_packed_refs_path(repo), "rt", encoding="UTF-8") as fp: for line in fp: @@ -133,8 +139,8 @@ def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: continue # END parse comment - # skip dereferenced tag object entries - previous line was actual - # tag reference for it + # Skip dereferenced tag object entries - previous line was actual + # tag reference for it. if line[0] == "^": continue @@ -153,7 +159,9 @@ def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> """ :return: hexsha stored in the reference at the given ref_path, recursively dereferencing all intermediate references as required - :param repo: the repository containing the reference at ref_path""" + + :param repo: The repository containing the reference at ref_path + """ while True: hexsha, ref_path = cls._get_ref_info(repo, ref_path) @@ -163,7 +171,11 @@ def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> @staticmethod def _check_ref_name_valid(ref_path: PathLike) -> None: - # Based on the rules described in https://git-scm.com/docs/git-check-ref-format/#_description + """Check a ref name for validity. + + This is based on the rules described in: + https://git-scm.com/docs/git-check-ref-format/#_description + """ previous: Union[str, None] = None one_before_previous: Union[str, None] = None for c in str(ref_path): @@ -210,9 +222,12 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: def _get_ref_info_helper( cls, repo: "Repo", ref_path: Union[PathLike, None] ) -> Union[Tuple[str, None], Tuple[None, str]]: - """Return: (str(sha), str(target_ref_path)) if available, the sha the file at - rela_path points to, or None. target_ref_path is the reference we - point to, or None""" + """ + :return: (str(sha), str(target_ref_path)) if available, the sha the file at + rela_path points to, or None. + + target_ref_path is the reference we point to, or None. + """ if ref_path: cls._check_ref_name_valid(ref_path) @@ -221,18 +236,18 @@ def _get_ref_info_helper( try: with open(os.path.join(repodir, str(ref_path)), "rt", encoding="UTF-8") as fp: value = fp.read().rstrip() - # Don't only split on spaces, but on whitespace, which allows to parse lines like + # Don't only split on spaces, but on whitespace, which allows to parse lines like: # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo tokens = value.split() assert len(tokens) != 0 except OSError: - # Probably we are just packed, find our entry in the packed refs file + # Probably we are just packed. Find our entry in the packed refs file. # NOTE: We are not a symbolic ref if we are in a packed file, as these - # are excluded explicitly + # are excluded explicitly. for sha, path in cls._iter_packed_refs(repo): if path != ref_path: continue - # sha will be used + # sha will be used. tokens = sha, path break # END for each packed ref @@ -240,11 +255,11 @@ def _get_ref_info_helper( if tokens is None: raise ValueError("Reference at %r does not exist" % ref_path) - # is it a reference ? + # Is it a reference? if tokens[0] == "ref:": return (None, tokens[1]) - # its a commit + # It's a commit. if repo.re_hexsha_only.match(tokens[0]): return (tokens[0], None) @@ -252,25 +267,31 @@ def _get_ref_info_helper( @classmethod def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[Tuple[str, None], Tuple[None, str]]: - """Return: (str(sha), str(target_ref_path)) if available, the sha the file at - rela_path points to, or None. target_ref_path is the reference we - point to, or None""" + """ + :return: (str(sha), str(target_ref_path)) if available, the sha the file at + rela_path points to, or None. + + target_ref_path is the reference we point to, or None. + """ return cls._get_ref_info_helper(repo, ref_path) def _get_object(self) -> Commit_ish: """ :return: The object our ref currently refers to. Refs can be cached, they will - always point to the actual object as it gets re-created on each query""" - # have to be dynamic here as we may be a tag which can point to anything - # Our path will be resolved to the hexsha which will be used accordingly + always point to the actual object as it gets re-created on each query. + """ + # We have to be dynamic here as we may be a tag which can point to anything. + # Our path will be resolved to the hexsha which will be used accordingly. return Object.new_from_sha(self.repo, hex_to_bin(self.dereference_recursive(self.repo, self.path))) def _get_commit(self) -> "Commit": """ :return: - Commit object we point to, works for detached and non-detached - SymbolicReferences. The symbolic reference will be dereferenced recursively.""" + Commit object we point to. This works for detached and non-detached + :class:`SymbolicReference` instances. The symbolic reference will be + dereferenced recursively. + """ obj = self._get_object() if obj.type == "tag": obj = obj.object @@ -286,12 +307,13 @@ def set_commit( commit: Union[Commit, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """As set_object, but restricts the type of object to be a Commit + """As set_object, but restricts the type of object to be a Commit. :raise ValueError: If commit is not a Commit object or doesn't point to a commit - :return: self""" - # check the type - assume the best if it is a base-string + :return: self + """ + # Check the type - assume the best if it is a base-string. invalid_type = False if isinstance(commit, Object): invalid_type = commit.type != Commit.type @@ -309,7 +331,7 @@ def set_commit( raise ValueError("Need commit, got %r" % commit) # END handle raise - # we leave strings to the rev-parse method below + # We leave strings to the rev-parse method below. self.set_object(commit, logmsg) return self @@ -320,14 +342,18 @@ def set_object( logmsg: Union[str, None] = None, ) -> "SymbolicReference": """Set the object we point to, possibly dereference our symbolic reference first. - If the reference does not exist, it will be created + If the reference does not exist, it will be created. - :param object: a refspec, a SymbolicReference or an Object instance. SymbolicReferences - will be dereferenced beforehand to obtain the object they point to + :param object: A refspec, a :class:`SymbolicReference` or an + :class:`Object ` instance. + :class:`SymbolicReference` instances will be dereferenced beforehand to + obtain the object they point to. :param logmsg: If not None, the message will be used in the reflog entry to be - written. Otherwise the reflog is not altered - :note: plain SymbolicReferences may not actually point to objects by convention - :return: self""" + written. Otherwise the reflog is not altered. + :note: Plain :class:`SymbolicReference` instances may not actually point to + objects by convention. + :return: self + """ if isinstance(object, SymbolicReference): object = object.object # @ReservedAssignment # END resolve references @@ -349,7 +375,9 @@ def set_object( object = property(_get_object, set_object, doc="Return the object our ref currently refers to") # type: ignore def _get_reference(self) -> "SymbolicReference": - """:return: Reference Object we point to + """ + :return: Reference Object we point to + :raise TypeError: If this symbolic reference is detached, hence it doesn't point to a reference, but to a commit""" sha, target_ref_path = self._get_ref_info(self.repo, self.path) @@ -367,18 +395,23 @@ def set_reference( will be set which effectively detaches the reference if it was a purely symbolic one. - :param ref: SymbolicReference instance, Object instance or refspec string - Only if the ref is a SymbolicRef instance, we will point to it. Everything - else is dereferenced to obtain the actual object. + :param ref: + A :class:`SymbolicReference` instance, + an :class:`Object ` instance, or a refspec string. + Only if the ref is a :class:`SymbolicReference` instance, we will point to + it. Everything else is dereferenced to obtain the actual object. + :param logmsg: If set to a string, the message will be used in the reflog. Otherwise, a reflog entry is not written for the changed reference. The previous commit of the entry will be the commit we point to now. - See also: log_append() + See also: :meth:`log_append` :return: self + :note: This symbolic reference will not be dereferenced. For that, see - ``set_object(...)``""" + :meth:`set_object`. + """ write_value = None obj = None if isinstance(ref, SymbolicReference): @@ -388,7 +421,7 @@ def set_reference( write_value = ref.hexsha elif isinstance(ref, str): try: - obj = self.repo.rev_parse(ref + "^{}") # optionally deref tags + obj = self.repo.rev_parse(ref + "^{}") # Optionally dereference tags. write_value = obj.hexsha except (BadObject, BadName) as e: raise ValueError("Could not extract object from %s" % ref) from e @@ -428,7 +461,7 @@ def set_reference( return self - # aliased reference + # Aliased reference reference: Union["Head", "TagReference", "RemoteReference", "Reference"] reference = property(_get_reference, set_reference, doc="Returns the Reference we point to") # type: ignore ref = reference @@ -437,7 +470,8 @@ def is_valid(self) -> bool: """ :return: True if the reference is valid, hence it can be read and points to - a valid object or reference.""" + a valid object or reference. + """ try: self.object except (OSError, ValueError): @@ -450,7 +484,8 @@ def is_detached(self) -> bool: """ :return: True if we are a detached reference, hence we point to a specific commit - instead to another reference""" + instead to another reference. + """ try: self.ref return False @@ -460,10 +495,11 @@ def is_detached(self) -> bool: def log(self) -> "RefLog": """ :return: RefLog for this reference. Its last entry reflects the latest change - applied to this reference + applied to this reference. .. note:: As the log is parsed every time, its recommended to cache it for use - instead of calling this method repeatedly. It should be considered read-only.""" + instead of calling this method repeatedly. It should be considered read-only. + """ return RefLog.from_file(RefLog.path(self)) def log_append( @@ -472,16 +508,17 @@ def log_append( message: Union[str, None], newbinsha: Union[bytes, None] = None, ) -> "RefLogEntry": - """Append a logentry to the logfile of this ref + """Append a logentry to the logfile of this ref. - :param oldbinsha: binary sha this ref used to point to - :param message: A message describing the change + :param oldbinsha: Binary sha this ref used to point to. + :param message: A message describing the change. :param newbinsha: The sha the ref points to now. If None, our current commit sha - will be used - :return: added RefLogEntry instance""" - # NOTE: we use the committer of the currently active commit - this should be + will be used. + :return: The added :class:`RefLogEntry ` instance. + """ + # NOTE: We use the committer of the currently active commit - this should be # correct to allow overriding the committer on a per-commit level. - # See https://github.com/gitpython-developers/GitPython/pull/146 + # See https://github.com/gitpython-developers/GitPython/pull/146. try: committer_or_reader: Union["Actor", "GitConfigParser"] = self.commit.committer except ValueError: @@ -496,19 +533,24 @@ def log_append( return RefLog.append_entry(committer_or_reader, RefLog.path(self), oldbinsha, newbinsha, message) def log_entry(self, index: int) -> "RefLogEntry": - """:return: RefLogEntry at the given index - :param index: python list compatible positive or negative index + """ + :return: RefLogEntry at the given index + + :param index: Python list compatible positive or negative index .. note:: This method must read part of the reflog during execution, hence - it should be used sparringly, or only if you need just one index. - In that case, it will be faster than the ``log()`` method""" + it should be used sparingly, or only if you need just one index. + In that case, it will be faster than the ``log()`` method. + """ return RefLog.entry_at(RefLog.path(self), index) @classmethod def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: """ :return: string with a full repository-relative path which can be used to initialize - a Reference instance, for instance by using ``Reference.from_path``""" + a Reference instance, for instance by using + :meth:`Reference.from_path `. + """ if isinstance(path, SymbolicReference): path = path.path full_ref_path = path @@ -520,21 +562,22 @@ def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: @classmethod def delete(cls, repo: "Repo", path: PathLike) -> None: - """Delete the reference at the given path + """Delete the reference at the given path. :param repo: - Repository to delete the reference from + Repository to delete the reference from. :param path: - Short or full path pointing to the reference, i.e. refs/myreference - or just "myreference", hence 'refs/' is implied. - Alternatively the symbolic reference to be deleted""" + Short or full path pointing to the reference, e.g. ``refs/myreference`` + or just ``myreference``, hence ``refs/`` is implied. + Alternatively the symbolic reference to be deleted. + """ full_ref_path = cls.to_full_path(path) abs_path = os.path.join(repo.common_dir, full_ref_path) if os.path.exists(abs_path): os.remove(abs_path) else: - # check packed refs + # Check packed refs. pack_file_path = cls._get_packed_refs_path(repo) try: with open(pack_file_path, "rb") as reader: @@ -545,10 +588,10 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: line = line_bytes.decode(defenc) _, _, line_ref = line.partition(" ") line_ref = line_ref.strip() - # keep line if it is a comment or if the ref to delete is not - # in the line + # Keep line if it is a comment or if the ref to delete is not + # in the line. # If we deleted the last line and this one is a tag-reference object, - # we drop it as well + # we drop it as well. if (line.startswith("#") or full_ref_path != line_ref) and ( not dropped_last_line or dropped_last_line and not line.startswith("^") ): @@ -557,21 +600,21 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: continue # END skip comments and lines without our path - # drop this line + # Drop this line. made_change = True dropped_last_line = True - # write the new lines + # Write the new lines. if made_change: - # write-binary is required, otherwise windows will - # open the file in text mode and change LF to CRLF ! + # Binary writing is required, otherwise Windows will + # open the file in text mode and change LF to CRLF! with open(pack_file_path, "wb") as fd: fd.writelines(line.encode(defenc) for line in new_lines) except OSError: - pass # it didn't exist at all + pass # It didn't exist at all. - # delete the reflog + # Delete the reflog. reflog_path = RefLog.path(cls(repo, full_ref_path)) if os.path.isfile(reflog_path): os.remove(reflog_path) @@ -587,16 +630,18 @@ def _create( force: bool, logmsg: Union[str, None] = None, ) -> T_References: - """internal method used to create a new symbolic reference. - If resolve is False, the reference will be taken as is, creating + """Internal method used to create a new symbolic reference. + + If `resolve` is False, the reference will be taken as is, creating a proper symbolic reference. Otherwise it will be resolved to the corresponding object and a detached symbolic reference will be created - instead""" + instead. + """ git_dir = _git_dir(repo, path) full_ref_path = cls.to_full_path(path) abs_ref_path = os.path.join(git_dir, full_ref_path) - # figure out target data + # Figure out target data. target = reference if resolve: target = repo.rev_parse(str(reference)) @@ -630,22 +675,22 @@ def create( force: bool = False, **kwargs: Any, ) -> T_References: - """Create a new symbolic reference, hence a reference pointing , to another reference. + """Create a new symbolic reference: a reference pointing to another reference. :param repo: - Repository to create the reference in + Repository to create the reference in. :param path: - full path at which the new symbolic reference is supposed to be - created at, i.e. "NEW_HEAD" or "symrefs/my_new_symref" + Full path at which the new symbolic reference is supposed to be + created at, e.g. ``NEW_HEAD`` or ``symrefs/my_new_symref``. :param reference: - The reference to which the new symbolic reference should point to. + The reference which the new symbolic reference should point to. If it is a commit-ish, the symbolic ref will be detached. :param force: - if True, force creation even if a symbolic reference with that name already exists. - Raise OSError otherwise + If True, force creation even if a symbolic reference with that name already exists. + Raise :class:`OSError` otherwise. :param logmsg: If not None, the message to append to the reflog. Otherwise no reflog @@ -657,23 +702,26 @@ def create( If a (Symbolic)Reference with the same name but different contents already exists. - :note: This does not alter the current HEAD, index or Working Tree""" + :note: This does not alter the current HEAD, index or working tree. + """ return cls._create(repo, path, cls._resolve_ref_on_create, reference, force, logmsg) def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference": - """Rename self to a new path + """Rename self to a new path. :param new_path: - Either a simple name or a full path, i.e. new_name or features/new_name. - The prefix refs/ is implied for references and will be set as needed. - In case this is a symbolic ref, there is no implied prefix + Either a simple name or a full path, e.g. ``new_name`` or ``features/new_name``. + The prefix ``refs/`` is implied for references and will be set as needed. + In case this is a symbolic ref, there is no implied prefix. :param force: If True, the rename will succeed even if a head with the target name - already exists. It will be overwritten in that case + already exists. It will be overwritten in that case. :return: self - :raise OSError: In case a file at path but a different contents already exists""" + + :raise OSError: If a file at path but with different contents already exists. + """ new_path = self.to_full_path(new_path) if self.path == new_path: return self @@ -682,15 +730,15 @@ def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference" cur_abs_path = os.path.join(_git_dir(self.repo, self.path), self.path) if os.path.isfile(new_abs_path): if not force: - # if they point to the same file, its not an error + # If they point to the same file, it's not an error. with open(new_abs_path, "rb") as fd1: f1 = fd1.read().strip() with open(cur_abs_path, "rb") as fd2: f2 = fd2.read().strip() if f1 != f2: raise OSError("File at path %r already exists" % new_abs_path) - # else: we could remove ourselves and use the otherone, but - # but clarity we just continue as usual + # else: We could remove ourselves and use the other one, but... + # ...for clarity, we just continue as usual. # END not force handling os.remove(new_abs_path) # END handle existing target file @@ -713,10 +761,10 @@ def _iter_items( common_path = cls._common_path_default rela_paths = set() - # walk loose refs - # Currently we do not follow links + # Walk loose refs. + # Currently we do not follow links. for root, dirs, files in os.walk(join_path_native(repo.common_dir, common_path)): - if "refs" not in root.split(os.sep): # skip non-refs subfolders + if "refs" not in root.split(os.sep): # Skip non-refs subfolders. refs_id = [d for d in dirs if d == "refs"] if refs_id: dirs[0:] = ["refs"] @@ -730,14 +778,14 @@ def _iter_items( # END for each file in root directory # END for each directory to walk - # read packed refs + # Read packed refs. for _sha, rela_path in cls._iter_packed_refs(repo): if rela_path.startswith(str(common_path)): rela_paths.add(rela_path) # END relative path matches common path # END packed refs reading - # return paths in sorted order + # Yield paths in sorted order. for path in sorted(rela_paths): try: yield cls.from_path(repo, path) @@ -753,37 +801,45 @@ def iter_items( *args: Any, **kwargs: Any, ) -> Iterator[T_References]: - """Find all refs in the repository + """Find all refs in the repository. :param repo: is the Repo :param common_path: - Optional keyword argument to the path which is to be shared by all - returned Ref objects. - Defaults to class specific portion if None assuring that only - refs suitable for the actual class are returned. + Optional keyword argument to the path which is to be shared by all returned + Ref objects. + Defaults to class specific portion if None, ensuring that only refs suitable + for the actual class are returned. :return: - git.SymbolicReference[], each of them is guaranteed to be a symbolic - ref which is not detached and pointing to a valid ref + A list of :class:`SymbolicReference`, each guaranteed to be a symbolic ref + which is not detached and pointing to a valid ref. - List is lexicographically sorted - The returned objects represent actual subclasses, such as Head or TagReference""" + The list is lexicographically sorted. The returned objects are instances of + concrete subclasses, such as :class:`Head ` or + :class:`TagReference `. + """ return (r for r in cls._iter_items(repo, common_path) if r.__class__ is SymbolicReference or not r.is_detached) @classmethod def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_References: """ - :param path: full .git-directory-relative path name to the Reference to instantiate - :note: use to_full_path() if you only have a partial path of a known Reference Type + Make a symbolic reference from a path. + + :param path: Full ``.git``-directory-relative path name to the Reference to instantiate. + + :note: Use :meth:`to_full_path` if you only have a partial path of a known Reference type. + :return: - Instance of type Reference, Head, or Tag - depending on the given path""" + Instance of type :class:`Reference `, + :class:`Head `, or :class:`Tag `, + depending on the given path. + """ if not path: raise ValueError("Cannot create Reference from %r" % path) - # Names like HEAD are inserted after the refs module is imported - we have an import dependency - # cycle and don't want to import these names in-function + # Names like HEAD are inserted after the refs module is imported - we have an + # import dependency cycle and don't want to import these names in-function. from . import HEAD, Head, RemoteReference, TagReference, Reference for ref_type in ( diff --git a/git/refs/tag.py b/git/refs/tag.py index d32d91bcf..3c269d9ba 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -18,10 +18,9 @@ class TagReference(Reference): - - """Class representing a lightweight tag reference which either points to a commit - ,a tag object or any other object. In the latter case additional information, - like the signature or the tag-creator, is available. + """A lightweight tag reference which either points to a commit, a tag object or any + other object. In the latter case additional information, like the signature or the + tag-creator, is available. This tag object will always point to a commit object, but may carry additional information in a tag object:: @@ -29,9 +28,11 @@ class TagReference(Reference): tagref = TagReference.list_items(repo)[0] print(tagref.commit.message) if tagref.tag is not None: - print(tagref.tag.message)""" + print(tagref.tag.message) + """ __slots__ = () + _common_default = "tags" _common_path_default = Reference._common_path_default + "/" + _common_default @@ -39,11 +40,12 @@ class TagReference(Reference): def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelated commit method """:return: Commit object the tag ref points to - :raise ValueError: if the tag points to a tree or blob""" + :raise ValueError: If the tag points to a tree or blob + """ obj = self.object while obj.type != "commit": if obj.type == "tag": - # it is a tag object which carries the commit as an object - we can point to anything + # It is a tag object which carries the commit as an object - we can point to anything. obj = obj.object else: raise ValueError( @@ -59,16 +61,13 @@ def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelat def tag(self) -> Union["TagObject", None]: """ :return: Tag object this tag ref points to or None in case - we are a light weight tag""" + we are a lightweight tag""" obj = self.object if obj.type == "tag": return obj return None - # make object read-only - # It should be reasonably hard to adjust an existing tag - - # object = property(Reference._get_object) + # Make object read-only. It should be reasonably hard to adjust an existing tag. @property def object(self) -> Commit_ish: # type: ignore[override] return Reference._get_object(self) @@ -86,30 +85,31 @@ def create( """Create a new tag reference. :param path: - The name of the tag, i.e. 1.0 or releases/1.0. - The prefix refs/tags is implied + The name of the tag, e.g. ``1.0`` or ``releases/1.0``. + The prefix ``refs/tags`` is implied. :param ref: - A reference to the Object you want to tag. The Object can be a commit, tree or - blob. + A reference to the :class:`Object ` you want to + tag. The Object can be a commit, tree or blob. :param logmsg: If not None, the message will be used in your tag object. This will also - create an additional tag object that allows to obtain that information, i.e.:: + create an additional tag object that allows to obtain that information, e.g.:: tagref.tag.message :param message: - Synonym for :param logmsg: - Included for backwards compatibility. :param logmsg is used in preference if both given. + Synonym for the `logmsg` parameter. + Included for backwards compatibility. `logmsg` takes precedence if both are passed. :param force: - If True, to force creation of a tag even though that tag already exists. + If True, force creation of a tag even though that tag already exists. :param kwargs: - Additional keyword arguments to be passed to git-tag + Additional keyword arguments to be passed to git-tag. - :return: A new TagReference""" + :return: A new TagReference. + """ if "ref" in kwargs and kwargs["ref"]: reference = kwargs["ref"] @@ -130,9 +130,9 @@ def create( @classmethod def delete(cls, repo: "Repo", *tags: "TagReference") -> None: # type: ignore[override] - """Delete the given existing tag or tags""" + """Delete the given existing tag or tags.""" repo.git.tag("-d", *tags) -# provide an alias +# Provide an alias. Tag = TagReference diff --git a/git/remote.py b/git/remote.py index fc2b2ceba..76352087a 100644 --- a/git/remote.py +++ b/git/remote.py @@ -4,7 +4,8 @@ # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ -# Module implementing a remote object allowing easy access to git remotes +"""Module implementing a remote object allowing easy access to git remotes.""" + import logging import re @@ -80,9 +81,13 @@ def add_progress( progress: Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None], ) -> Any: """Add the --progress flag to the given kwargs dict if supported by the - git command. If the actual progress in the given progress instance is not - given, we do not request any progress - :return: possibly altered kwargs""" + git command. + + :note: If the actual progress in the given progress instance is not + given, we do not request any progress. + + :return: possibly altered kwargs + """ if progress is not None: v = git.version_info[:2] if v >= (1, 7): @@ -113,18 +118,16 @@ def to_progress_instance(progress: RemoteProgress) -> RemoteProgress: def to_progress_instance( progress: Union[Callable[..., Any], RemoteProgress, None] ) -> Union[RemoteProgress, CallableRemoteProgress]: - """Given the 'progress' return a suitable object derived from - RemoteProgress(). - """ - # new API only needs progress as a function + """Given the 'progress' return a suitable object derived from RemoteProgress.""" + # New API only needs progress as a function. if callable(progress): return CallableRemoteProgress(progress) - # where None is passed create a parser that eats the progress + # Where None is passed create a parser that eats the progress. elif progress is None: return RemoteProgress() - # assume its the old API with an instance of RemoteProgress. + # Assume its the old API with an instance of RemoteProgress. return progress @@ -152,6 +155,7 @@ class PushInfo(IterableObj, object): "_remote", "summary", ) + _id_attribute_ = "pushinfo" ( @@ -187,8 +191,10 @@ def __init__( old_commit: Optional[str] = None, summary: str = "", ) -> None: - """Initialize a new instance - local_ref: HEAD | Head | RemoteReference | TagReference | Reference | SymbolicReference | None""" + """Initialize a new instance. + + local_ref: HEAD | Head | RemoteReference | TagReference | Reference | SymbolicReference | None + """ self.flags = flags self.local_ref = local_ref self.remote_ref_string = remote_ref_string @@ -204,9 +210,11 @@ def old_commit(self) -> Union[str, SymbolicReference, Commit_ish, None]: def remote_ref(self) -> Union[RemoteReference, TagReference]: """ :return: - Remote Reference or TagReference in the local repository corresponding - to the remote_ref_string kept in this instance.""" - # translate heads to a local remote, tags stay as they are + Remote :class:`Reference ` or + :class:`TagReference ` in the local repository + corresponding to the :attr:`remote_ref_string` kept in this instance. + """ + # Translate heads to a local remote. Tags stay as they are. if self.remote_ref_string.startswith("refs/tags"): return TagReference(self._remote.repo, self.remote_ref_string) elif self.remote_ref_string.startswith("refs/heads"): @@ -222,11 +230,11 @@ def remote_ref(self) -> Union[RemoteReference, TagReference]: @classmethod def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": """Create a new PushInfo instance as parsed from line which is expected to be like - refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes""" + refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes.""" control_character, from_to, summary = line.split("\t", 3) flags = 0 - # control character handling + # Control character handling try: flags |= cls._flag_map[control_character] except KeyError as e: @@ -243,7 +251,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": else: from_ref = Reference.from_path(remote.repo, from_ref_string) - # commit handling, could be message or commit info + # Commit handling, could be message or commit info old_commit: Optional[str] = None if summary.startswith("["): if "[rejected]" in summary: @@ -260,13 +268,13 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": flags |= cls.NEW_HEAD # uptodate encoded in control character else: - # fast-forward or forced update - was encoded in control character, - # but we parse the old and new commit + # Fast-forward or forced update - was encoded in control character, + # but we parse the old and new commit. split_token = "..." if control_character == " ": split_token = ".." old_sha, _new_sha = summary.split(" ")[0].split(split_token) - # have to use constructor here as the sha usually is abbreviated + # Have to use constructor here as the sha usually is abbreviated. old_commit = old_sha # END message handling @@ -278,9 +286,7 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> class PushInfoList(IterableList[PushInfo]): - """ - IterableList of PushInfo objects. - """ + """IterableList of PushInfo objects.""" def __new__(cls) -> "PushInfoList": return cast(PushInfoList, IterableList.__new__(cls, "push_infos")) @@ -290,15 +296,12 @@ def __init__(self) -> None: self.error: Optional[Exception] = None def raise_if_error(self) -> None: - """ - Raise an exception if any ref failed to push. - """ + """Raise an exception if any ref failed to push.""" if self.error: raise self.error class FetchInfo(IterableObj, object): - """ Carries information about the results of a fetch operation of a single head:: @@ -315,6 +318,7 @@ class FetchInfo(IterableObj, object): """ __slots__ = ("ref", "old_commit", "flags", "note", "remote_ref_path") + _id_attribute_ = "fetchinfo" ( @@ -341,9 +345,7 @@ class FetchInfo(IterableObj, object): @classmethod def refresh(cls) -> Literal[True]: - """This gets called by the refresh function (see the top level - __init__). - """ + """This gets called by the refresh function (see the top level __init__).""" # clear the old values in _flag_map try: del cls._flag_map["t"] @@ -371,9 +373,7 @@ def __init__( old_commit: Union[Commit_ish, None] = None, remote_ref_path: Optional[PathLike] = None, ) -> None: - """ - Initialize a new instance - """ + """Initialize a new instance.""" self.ref = ref self.flags = flags self.note = note @@ -410,12 +410,13 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": ' ' means a fast-forward fetch line is the corresponding line from FETCH_HEAD, like - acb0fa8b94ef421ad60c8507b634759a472cd56c not-for-merge branch '0.1.7RC' of /tmp/tmpya0vairemote_repo""" + acb0fa8b94ef421ad60c8507b634759a472cd56c not-for-merge branch '0.1.7RC' of /tmp/tmpya0vairemote_repo + """ match = cls._re_fetch_result.match(line) if match is None: raise ValueError("Failed to parse line: %r" % line) - # parse lines + # Parse lines. remote_local_ref_str: str ( control_character, @@ -432,7 +433,7 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": except ValueError as e: # unpack error raise ValueError("Failed to parse FETCH_HEAD line: %r" % fetch_line) from e - # parse flags from control_character + # Parse flags from control_character. flags = 0 try: flags |= cls._flag_map[control_character] @@ -440,7 +441,8 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": raise ValueError("Control character %r unknown as parsed from line %r" % (control_character, line)) from e # END control char exception handling - # parse operation string for more info - makes no sense for symbolic refs, but we parse it anyway + # Parse operation string for more info. + # This makes no sense for symbolic refs, but we parse it anyway. old_commit: Union[Commit_ish, None] = None is_tag_operation = False if "rejected" in operation: @@ -460,45 +462,45 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": old_commit = repo.rev_parse(operation.split(split_token)[0]) # END handle refspec - # handle FETCH_HEAD and figure out ref type + # Handle FETCH_HEAD and figure out ref type. # If we do not specify a target branch like master:refs/remotes/origin/master, # the fetch result is stored in FETCH_HEAD which destroys the rule we usually - # have. In that case we use a symbolic reference which is detached + # have. In that case we use a symbolic reference which is detached. ref_type: Optional[Type[SymbolicReference]] = None if remote_local_ref_str == "FETCH_HEAD": ref_type = SymbolicReference elif ref_type_name == "tag" or is_tag_operation: - # the ref_type_name can be branch, whereas we are still seeing a tag operation. It happens during - # testing, which is based on actual git operations + # The ref_type_name can be branch, whereas we are still seeing a tag operation. + # It happens during testing, which is based on actual git operations. ref_type = TagReference elif ref_type_name in ("remote-tracking", "branch"): - # note: remote-tracking is just the first part of the 'remote-tracking branch' token. - # We don't parse it correctly, but its enough to know what to do, and its new in git 1.7something + # Note: remote-tracking is just the first part of the 'remote-tracking branch' token. + # We don't parse it correctly, but its enough to know what to do, and it's new in git 1.7something. ref_type = RemoteReference elif "/" in ref_type_name: - # If the fetch spec look something like this '+refs/pull/*:refs/heads/pull/*', and is thus pretty - # much anything the user wants, we will have trouble to determine what's going on - # For now, we assume the local ref is a Head + # If the fetch spec look something like this '+refs/pull/*:refs/heads/pull/*', + # and is thus pretty much anything the user wants, we will have trouble + # determining what's going on. For now, we assume the local ref is a Head. ref_type = Head else: raise TypeError("Cannot handle reference type: %r" % ref_type_name) # END handle ref type - # create ref instance + # Create ref instance. if ref_type is SymbolicReference: remote_local_ref = ref_type(repo, "FETCH_HEAD") else: - # determine prefix. Tags are usually pulled into refs/tags, they may have subdirectories. - # It is not clear sometimes where exactly the item is, unless we have an absolute path as indicated - # by the 'ref/' prefix. Otherwise even a tag could be in refs/remotes, which is when it will have the - # 'tags/' subdirectory in its path. + # Determine prefix. Tags are usually pulled into refs/tags, they may have subdirectories. + # It is not clear sometimes where exactly the item is, unless we have an absolute path as + # indicated by the 'ref/' prefix. Otherwise even a tag could be in refs/remotes, which is + # when it will have the 'tags/' subdirectory in its path. # We don't want to test for actual existence, but try to figure everything out analytically. ref_path: Optional[PathLike] = None remote_local_ref_str = remote_local_ref_str.strip() if remote_local_ref_str.startswith(Reference._common_path_default + "/"): - # always use actual type if we get absolute paths - # Will always be the case if something is fetched outside of refs/remotes (if its not a tag) + # Always use actual type if we get absolute paths. + # Will always be the case if something is fetched outside of refs/remotes (if its not a tag). ref_path = remote_local_ref_str if ref_type is not TagReference and not remote_local_ref_str.startswith( RemoteReference._common_path_default + "/" @@ -506,14 +508,14 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": ref_type = Reference # END downgrade remote reference elif ref_type is TagReference and "tags/" in remote_local_ref_str: - # even though its a tag, it is located in refs/remotes + # Even though it's a tag, it is located in refs/remotes. ref_path = join_path(RemoteReference._common_path_default, remote_local_ref_str) else: ref_path = join_path(ref_type._common_path_default, remote_local_ref_str) # END obtain refpath - # even though the path could be within the git conventions, we make - # sure we respect whatever the user wanted, and disabled path checking + # Even though the path could be within the git conventions, we make + # sure we respect whatever the user wanted, and disabled path checking. remote_local_ref = ref_type(repo, ref_path, check_path=False) # END create ref instance @@ -527,16 +529,17 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> class Remote(LazyMixin, IterableObj): - """Provides easy read and write access to a git remote. Everything not part of this interface is considered an option for the current remote, allowing constructs like remote.pushurl to query the pushurl. - NOTE: When querying configuration, the configuration accessor will be cached - to speed up subsequent accesses.""" + :note: When querying configuration, the configuration accessor will be cached + to speed up subsequent accesses. + """ __slots__ = ("repo", "name", "_config_reader") + _id_attribute_ = "name" unsafe_git_fetch_options = [ @@ -557,22 +560,23 @@ class Remote(LazyMixin, IterableObj): ] def __init__(self, repo: "Repo", name: str) -> None: - """Initialize a remote instance + """Initialize a remote instance. :param repo: The repository we are a remote of - :param name: the name of the remote, i.e. 'origin'""" + :param name: The name of the remote, e.g. 'origin' + """ self.repo = repo self.name = name self.url: str def __getattr__(self, attr: str) -> Any: """Allows to call this instance like - remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name""" + remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name.""" if attr == "_config_reader": return super(Remote, self).__getattr__(attr) - # sometimes, probably due to a bug in python itself, we are being called - # even though a slot of the same name exists + # Sometimes, probably due to a bug in Python itself, we are being called + # even though a slot of the same name exists. try: return self._config_reader.get(attr) except cp.NoOptionError: @@ -584,8 +588,8 @@ def _config_section_name(self) -> str: def _set_cache_(self, attr: str) -> None: if attr == "_config_reader": - # NOTE: This is cached as __getattr__ is overridden to return remote config values implicitly, such as - # in print(r.pushurl) + # NOTE: This is cached as __getattr__ is overridden to return remote config + # values implicitly, such as in print(r.pushurl). self._config_reader = SectionConstraint(self.repo.config_reader("repository"), self._config_section_name()) else: super(Remote, self)._set_cache_(attr) @@ -608,12 +612,13 @@ def __hash__(self) -> int: def exists(self) -> bool: """ :return: True if this is a valid, existing remote. - Valid remotes have an entry in the repository's configuration""" + Valid remotes have an entry in the repository's configuration. + """ try: self.config_reader.get("url") return True except cp.NoOptionError: - # we have the section at least ... + # We have the section at least... return True except cp.NoSectionError: return False @@ -635,12 +640,12 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote def set_url( self, new_url: str, old_url: Optional[str] = None, allow_unsafe_protocols: bool = False, **kwargs: Any ) -> "Remote": - """Configure URLs on current remote (cf command git remote set_url) + """Configure URLs on current remote (cf command git remote set_url). This command manages URLs on the remote. - :param new_url: string being the URL to add as an extra remote URL - :param old_url: when set, replaces this URL with new_url for the remote + :param new_url: String being the URL to add as an extra remote URL + :param old_url: When set, replaces this URL with new_url for the remote :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext :return: self """ @@ -655,12 +660,12 @@ def set_url( return self def add_url(self, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) -> "Remote": - """Adds a new url on current remote (special case of git remote set_url) + """Adds a new url on current remote (special case of git remote set_url). This command adds new URLs to a given remote, making it possible to have multiple URLs for a single remote. - :param url: string being the URL to add as an extra remote URL + :param url: String being the URL to add as an extra remote URL :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext :return: self """ @@ -672,7 +677,7 @@ def delete_url(self, url: str, **kwargs: Any) -> "Remote": This command deletes new URLs to a given remote, making it possible to have multiple URLs for a single remote. - :param url: string being the URL to delete from the remote + :param url: String being the URL to delete from the remote :return: self """ return self.set_url(url, delete=True) @@ -700,7 +705,7 @@ def urls(self) -> Iterator[str]: yield line.split(": ")[-1] except GitCommandError as _ex: if any(msg in str(_ex) for msg in ["correct access rights", "cannot run ssh"]): - # If ssh is not setup to access this repository, see issue 694 + # If ssh is not setup to access this repository, see issue 694. remote_details = self.repo.git.config("--get-all", "remote.%s.url" % self.name) assert isinstance(remote_details, str) for line in remote_details.split("\n"): @@ -715,8 +720,9 @@ def refs(self) -> IterableList[RemoteReference]: """ :return: IterableList of RemoteReference objects. It is prefixed, allowing - you to omit the remote path portion, i.e.:: - remote.refs.master # yields RemoteReference('/refs/remotes/origin/master')""" + you to omit the remote path portion, e.g.:: + remote.refs.master # yields RemoteReference('/refs/remotes/origin/master') + """ out_refs: IterableList[RemoteReference] = IterableList(RemoteReference._id_attribute_, "%s/" % self.name) out_refs.extend(RemoteReference.list_items(self.repo, remote=self.name)) return out_refs @@ -745,7 +751,7 @@ def stale_refs(self) -> IterableList[Reference]: if not line.startswith(token): continue ref_name = line.replace(token, "") - # sometimes, paths start with a full ref name, like refs/tags/foo, see #260 + # Sometimes, paths start with a full ref name, like refs/tags/foo. See #260. if ref_name.startswith(Reference._common_path_default + "/"): out_refs.append(Reference.from_path(self.repo, ref_name)) else: @@ -757,7 +763,7 @@ def stale_refs(self) -> IterableList[Reference]: @classmethod def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) -> "Remote": - """Create a new remote to the given repository + """Create a new remote to the given repository. :param repo: Repository instance that is to receive the new remote :param name: Desired name of the remote @@ -765,7 +771,8 @@ def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext :param kwargs: Additional arguments to be passed to the git-remote add command :return: New Remote instance - :raise GitCommandError: in case an origin with that name already exists""" + :raise GitCommandError: in case an origin with that name already exists + """ scmd = "add" kwargs["insert_kwargs_after"] = scmd url = Git.polish_url(url) @@ -774,29 +781,30 @@ def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool repo.git.remote(scmd, "--", name, url, **kwargs) return cls(repo, name) - # add is an alias + # `add` is an alias. @classmethod def add(cls, repo: "Repo", name: str, url: str, **kwargs: Any) -> "Remote": return cls.create(repo, name, url, **kwargs) @classmethod def remove(cls, repo: "Repo", name: str) -> str: - """Remove the remote with the given name + """Remove the remote with the given name. - :return: the passed remote name to remove + :return: The passed remote name to remove """ repo.git.remote("rm", name) if isinstance(name, cls): name._clear_cache() return name - # alias + # `rm` is an alias. rm = remove def rename(self, new_name: str) -> "Remote": - """Rename self to the given new_name + """Rename self to the given new_name. - :return: self""" + :return: self + """ if self.name == new_name: return self @@ -808,13 +816,12 @@ def rename(self, new_name: str) -> "Remote": def update(self, **kwargs: Any) -> "Remote": """Fetch all changes for this remote, including new branches which will - be forced in ( in case your local remote branch is not part the new remote branches - ancestry anymore ). - - :param kwargs: - Additional arguments passed to git-remote update + be forced in (in case your local remote branch is not part the new remote + branch's ancestry anymore). - :return: self""" + :param kwargs: Additional arguments passed to git-remote update + :return: self + """ scmd = "update" kwargs["insert_kwargs_after"] = scmd self.repo.git.remote(scmd, self.name, **kwargs) @@ -828,15 +835,15 @@ def _get_fetch_info_from_stderr( ) -> IterableList["FetchInfo"]: progress = to_progress_instance(progress) - # skip first line as it is some remote info we are not interested in + # Skip first line as it is some remote info we are not interested in. output: IterableList["FetchInfo"] = IterableList("name") - # lines which are no progress are fetch info lines - # this also waits for the command to finish - # Skip some progress lines that don't provide relevant information + # Lines which are no progress are fetch info lines. + # This also waits for the command to finish. + # Skip some progress lines that don't provide relevant information. fetch_info_lines = [] - # Basically we want all fetch info lines which appear to be in regular form, and thus have a - # command character. Everything else we ignore, + # Basically we want all fetch info lines which appear to be in regular form, and + # thus have a command character. Everything else we ignore. cmds = set(FetchInfo._flag_map.keys()) progress_handler = progress.new_message_handler() @@ -861,7 +868,7 @@ def _get_fetch_info_from_stderr( fetch_info_lines.append(line) continue - # read head information + # Read head information. fetch_head = SymbolicReference(self.repo, "FETCH_HEAD") with open(fetch_head.abspath, "rb") as fp: fetch_head_info = [line.decode(defenc) for line in fp.readlines()] @@ -899,10 +906,10 @@ def _get_push_info( ) -> PushInfoList: progress = to_progress_instance(progress) - # read progress information from stderr - # we hope stdout can hold all the data, it should ... - # read the lines manually as it will use carriage returns between the messages - # to override the previous one. This is why we read the bytes manually + # Read progress information from stderr. + # We hope stdout can hold all the data, it should... + # Read the lines manually as it will use carriage returns between the messages + # to override the previous one. This is why we read the bytes manually. progress_handler = progress.new_message_handler() output: PushInfoList = PushInfoList() @@ -925,8 +932,8 @@ def stdout_handler(line: str) -> None: try: proc.wait(stderr=stderr_text) except Exception as e: - # This is different than fetch (which fails if there is any std_err - # even if there is an output) + # This is different than fetch (which fails if there is any stderr + # even if there is an output). if not output: raise elif stderr_text: @@ -936,7 +943,7 @@ def stdout_handler(line: str) -> None: return output def _assert_refspec(self) -> None: - """Turns out we can't deal with remotes if the refspec is missing""" + """Turns out we can't deal with remotes if the refspec is missing.""" config = self.config_reader unset = "placeholder" try: @@ -958,38 +965,46 @@ def fetch( allow_unsafe_options: bool = False, **kwargs: Any, ) -> IterableList[FetchInfo]: - """Fetch the latest changes for this remote + """Fetch the latest changes for this remote. :param refspec: A "refspec" is used by fetch and push to describe the mapping between remote ref and local ref. They are combined with a colon in - the format :, preceded by an optional plus sign, +. - For example: git fetch $URL refs/heads/master:refs/heads/origin means + the format ``:``, preceded by an optional plus sign, ``+``. + For example: ``git fetch $URL refs/heads/master:refs/heads/origin`` means "grab the master branch head from the $URL and store it as my origin - branch head". And git push $URL refs/heads/master:refs/heads/to-upstream + branch head". And ``git push $URL refs/heads/master:refs/heads/to-upstream`` means "publish my master branch head as to-upstream branch at $URL". See also git-push(1). - Taken from the git manual + Taken from the git manual, gitglossary(7). Fetch supports multiple refspecs (as the underlying git-fetch does) - supplying a list rather than a string for 'refspec' will make use of this facility. - :param progress: See 'push' method - :param verbose: Boolean for verbose output + + :param progress: See :meth:`push` method. + + :param verbose: Boolean for verbose output. + :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process should be killed. It is set to None by default. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: Additional arguments to be passed to git-fetch + + :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + + :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + + :param kwargs: Additional arguments to be passed to git-fetch. + :return: IterableList(FetchInfo, ...) list of FetchInfo instances providing detailed information about the fetch results :note: As fetch does not provide progress information to non-ttys, we cannot make - it available here unfortunately as in the 'push' method.""" + it available here unfortunately as in the :meth:`push` method. + """ if refspec is None: # No argument refspec, then ensure the repo's config has a fetch refspec. self._assert_refspec() @@ -1028,13 +1043,14 @@ def pull( """Pull changes from the given branch, being the same as a fetch followed by a merge of branch with your local branch. - :param refspec: see :meth:`fetch` method - :param progress: see :meth:`push` method - :param kill_after_timeout: see :meth:`fetch` method + :param refspec: See :meth:`fetch` method + :param progress: See :meth:`push` method + :param kill_after_timeout: See :meth:`fetch` method :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack :param kwargs: Additional arguments to be passed to git-pull - :return: Please see :meth:`fetch` method""" + :return: Please see :meth:`fetch` method + """ if refspec is None: # No argument refspec, then ensure the repo's config has a fetch refspec. self._assert_refspec() @@ -1067,33 +1083,42 @@ def push( ) -> PushInfoList: """Push changes from source branch in refspec to target branch in refspec. - :param refspec: see 'fetch' method + :param refspec: See :meth:`fetch` method. + :param progress: Can take one of many value types: - * None to discard progress information + * None to discard progress information. * A function (callable) that is called with the progress information. Signature: ``progress(op_code, cur_count, max_count=None, message='')``. `Click here `__ for a description of all arguments given to the function. - * An instance of a class derived from ``git.RemoteProgress`` that - overrides the ``update()`` function. + * An instance of a class derived from :class:`git.RemoteProgress` that + overrides the :meth:`update ` method. :note: No further progress information is returned after push returns. + :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process should be killed. It is set to None by default. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --receive-pack - :param kwargs: Additional arguments to be passed to git-push + + :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + + :param allow_unsafe_options: + Allow unsafe options to be used, like --receive-pack. + + :param kwargs: Additional arguments to be passed to git-push. + :return: - A ``PushInfoList`` object, where each list member + A :class:`PushInfoList` object, where each list member represents an individual head which had been updated on the remote side. - If the push contains rejected heads, these will have the PushInfo.ERROR bit set - in their flags. + If the push contains rejected heads, these will have the + :attr:`PushInfo.ERROR` bit set in their flags. If the operation fails completely, the length of the returned PushInfoList will be 0. - Call ``.raise_if_error()`` on the returned object to raise on any failure.""" + Call :meth:`raise_if_error ` on the returned + object to raise on any failure. + """ kwargs = add_progress(kwargs, self.repo.git, progress) refspec = Git._unpack_args(refspec or []) @@ -1121,7 +1146,8 @@ def config_reader(self) -> SectionConstraint[GitConfigParser]: """ :return: GitConfigParser compatible object able to read options for only our remote. - Hence you may simple type config.get("pushurl") to obtain the information""" + Hence you may simple type config.get("pushurl") to obtain the information. + """ return self._config_reader def _clear_cache(self) -> None: @@ -1135,15 +1161,17 @@ def _clear_cache(self) -> None: def config_writer(self) -> SectionConstraint: """ :return: GitConfigParser compatible object able to write options for this remote. + :note: You can only own one writer at a time - delete it to release the configuration file and make it usable by others. To assure consistent results, you should only query options through the writer. Once you are done writing, you are free to use the config reader - once again.""" + once again. + """ writer = self.repo.config_writer() - # clear our cache to assure we re-read the possibly changed configuration + # Clear our cache to ensure we re-read the possibly changed configuration. self._clear_cache() return SectionConstraint(writer, self._config_section_name()) diff --git a/git/repo/__init__.py b/git/repo/__init__.py index 23c18db85..f1eac3311 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -1,3 +1,5 @@ -"""Initialize the Repo package""" +"""Initialize the Repo package.""" + # flake8: noqa + from .base import Repo as Repo diff --git a/git/repo/base.py b/git/repo/base.py index 23136a1d1..da81698d8 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -3,7 +3,9 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + from __future__ import annotations + import logging import os import re @@ -164,10 +166,10 @@ def __init__( search_parent_directories: bool = False, expand_vars: bool = True, ) -> None: - """Create a new Repo instance + """Create a new Repo instance. :param path: - the path to either the root git directory or the bare git repo:: + The path to either the root git directory or the bare git repo:: repo = Repo("/Users/mtrier/Development/git-python") repo = Repo("/Users/mtrier/Development/git-python.git") @@ -175,21 +177,26 @@ def __init__( repo = Repo("$REPOSITORIES/Development/git-python.git") repo = Repo("C:\\Users\\mtrier\\Development\\git-python\\.git") - - In *Cygwin*, path may be a `'cygdrive/...'` prefixed path. - - If it evaluates to false, :envvar:`GIT_DIR` is used, and if this also evals to false, - the current-directory is used. + - In *Cygwin*, path may be a ``cygdrive/...`` prefixed path. + - If it evaluates to false, :envvar:`GIT_DIR` is used, and if this also + evals to false, the current-directory is used. + :param odbt: Object DataBase type - a type which is constructed by providing the directory containing the database objects, i.e. .git/objects. It will be used to access all object data + :param search_parent_directories: - if True, all parent directories will be searched for a valid repo as well. + If True, all parent directories will be searched for a valid repo as well. + + Please note that this was the default behaviour in older versions of + GitPython, which is considered a bug though. - Please note that this was the default behaviour in older versions of GitPython, - which is considered a bug though. :raise InvalidGitRepositoryError: :raise NoSuchPathError: - :return: git.Repo""" + + :return: git.Repo + """ epath = path or os.getenv("GIT_DIR") if not epath: @@ -271,7 +278,7 @@ def __init__( try: self._bare = self.config_reader("repository").getboolean("core", "bare") except Exception: - # lets not assume the option exists, although it should + # Let's not assume the option exists, although it should. pass try: @@ -280,8 +287,8 @@ def __init__( except OSError: self._common_dir = "" - # adjust the wd in case we are actually bare - we didn't know that - # in the first place + # Adjust the working directory in case we are actually bare - we didn't know + # that in the first place. if self._bare: self._working_tree_dir = None # END working dir handling @@ -289,7 +296,7 @@ def __init__( self.working_dir: PathLike = self._working_tree_dir or self.common_dir self.git = self.GitCommandWrapperType(self.working_dir) - # special handling, in special times + # Special handling, in special times. rootpath = osp.join(self.common_dir, "objects") if issubclass(odbt, GitCmdObjectDB): self.odb = odbt(rootpath, self.git) @@ -311,12 +318,10 @@ def __del__(self) -> None: def close(self) -> None: if self.git: self.git.clear_cache() - # Tempfiles objects on Windows are holding references to - # open files until they are collected by the garbage - # collector, thus preventing deletion. - # TODO: Find these references and ensure they are closed - # and deleted synchronously rather than forcing a gc - # collection. + # Tempfiles objects on Windows are holding references to open files until + # they are collected by the garbage collector, thus preventing deletion. + # TODO: Find these references and ensure they are closed and deleted + # synchronously rather than forcing a gc collection. if is_win: gc.collect() gitdb.util.mman.collect() @@ -351,14 +356,18 @@ def _set_description(self, descr: str) -> None: @property def working_tree_dir(self) -> Optional[PathLike]: - """:return: The working tree directory of our git repository. If this is a bare repository, None is returned.""" + """ + :return: The working tree directory of our git repository. + If this is a bare repository, None is returned. + """ return self._working_tree_dir @property def common_dir(self) -> PathLike: """ :return: The git dir that holds everything except possibly HEAD, - FETCH_HEAD, ORIG_HEAD, COMMIT_EDITMSG, index, and logs/.""" + FETCH_HEAD, ORIG_HEAD, COMMIT_EDITMSG, index, and logs/. + """ return self._common_dir or self.git_dir @property @@ -368,30 +377,36 @@ def bare(self) -> bool: @property def heads(self) -> "IterableList[Head]": - """A list of ``Head`` objects representing the branch heads in - this repo + """A list of ``Head`` objects representing the branch heads in this repo. - :return: ``git.IterableList(Head, ...)``""" + :return: ``git.IterableList(Head, ...)`` + """ return Head.list_items(self) @property def references(self) -> "IterableList[Reference]": """A list of Reference objects representing tags, heads and remote references. - :return: IterableList(Reference, ...)""" + :return: IterableList(Reference, ...) + """ return Reference.list_items(self) - # alias for references + # Alias for references. refs = references - # alias for heads + # Alias for heads. branches = heads @property def index(self) -> "IndexFile": - """:return: IndexFile representing this repository's index. - :note: This property can be expensive, as the returned ``IndexFile`` will be - reinitialized. It's recommended to re-use the object.""" + """ + :return: :class:`IndexFile ` representing this + repository's index. + + :note: This property can be expensive, as the returned + :class:`IndexFile ` will be reinitialized. + It is recommended to reuse the object. + """ return IndexFile(self) @property @@ -401,14 +416,17 @@ def head(self) -> "HEAD": @property def remotes(self) -> "IterableList[Remote]": - """A list of Remote objects allowing to access and manipulate remotes + """A list of Remote objects allowing to access and manipulate remotes. - :return: ``git.IterableList(Remote, ...)``""" + :return: ``git.IterableList(Remote, ...)`` + """ return Remote.list_items(self) def remote(self, name: str = "origin") -> "Remote": """:return: Remote with the specified name - :raise ValueError: if no remote with such a name exists""" + + :raise ValueError: If no remote with such a name exists + """ r = Remote(self, name) if not r.exists(): raise ValueError("Remote named '%s' didn't exist" % name) @@ -420,12 +438,15 @@ def remote(self, name: str = "origin") -> "Remote": def submodules(self) -> "IterableList[Submodule]": """ :return: git.IterableList(Submodule, ...) of direct submodules - available from the current head""" + available from the current head + """ return Submodule.list_items(self) def submodule(self, name: str) -> "Submodule": """:return: Submodule with the given name - :raise ValueError: If no such submodule exists""" + + :raise ValueError: If no such submodule exists + """ try: return self.submodules[name] except IndexError as e: @@ -433,38 +454,47 @@ def submodule(self, name: str) -> "Submodule": # END exception handling def create_submodule(self, *args: Any, **kwargs: Any) -> Submodule: - """Create a new submodule + """Create a new submodule. :note: See the documentation of Submodule.add for a description of the - applicable parameters - :return: created submodules""" + applicable parameters. + + :return: The created submodules. + """ return Submodule.add(self, *args, **kwargs) def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """An iterator yielding Submodule instances, see Traversable interface - for a description of args and kwargs + for a description of args and kwargs. - :return: Iterator""" + :return: Iterator + """ return RootModule(self).traverse(*args, **kwargs) def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """Update the submodules, keeping the repository consistent as it will - take the previous state into consideration. For more information, please - see the documentation of RootModule.update""" + take the previous state into consideration. + + :note: For more information, please see the documentation of + :meth:`RootModule.update `. + """ return RootModule(self).update(*args, **kwargs) # }END submodules @property def tags(self) -> "IterableList[TagReference]": - """A list of ``Tag`` objects that are available in this repo + """A list of ``Tag`` objects that are available in this repo. - :return: ``git.IterableList(TagReference, ...)``""" + :return: ``git.IterableList(TagReference, ...)`` + """ return TagReference.list_items(self) def tag(self, path: PathLike) -> TagReference: """:return: TagReference Object, reference pointing to a Commit or Tag - :param path: path to the tag reference, i.e. 0.1.5 or tags/0.1.5""" + + :param path: path to the tag reference, i.e. 0.1.5 or tags/0.1.5 + """ full_path = self._to_full_tag_path(path) return TagReference(self, full_path) @@ -486,15 +516,19 @@ def create_head( logmsg: Optional[str] = None, ) -> "Head": """Create a new head within the repository. - For more documentation, please see the Head.create method. - :return: newly created Head Reference""" + :note: For more documentation, please see the + :meth:`Head.create ` method. + + :return: Newly created :class:`Head ` Reference + """ return Head.create(self, path, commit, logmsg, force) def delete_head(self, *heads: "Union[str, Head]", **kwargs: Any) -> None: - """Delete the given heads + """Delete the given heads. - :param kwargs: Additional keyword arguments to be passed to git-branch""" + :param kwargs: Additional keyword arguments to be passed to git-branch + """ return Head.delete(self, *heads, **kwargs) def create_tag( @@ -506,9 +540,12 @@ def create_tag( **kwargs: Any, ) -> TagReference: """Create a new tag reference. - For more documentation, please see the TagReference.create method. - :return: TagReference object""" + :note: For more documentation, please see the + :meth:`TagReference.create ` method. + + :return: :class:`TagReference ` object + """ return TagReference.create(self, path, ref, message, force, **kwargs) def delete_tag(self, *tags: TagReference) -> None: @@ -518,10 +555,11 @@ def delete_tag(self, *tags: TagReference) -> None: def create_remote(self, name: str, url: str, **kwargs: Any) -> Remote: """Create a new remote. - For more information, please see the documentation of the Remote.create - methods + For more information, please see the documentation of the + :meth:`Remote.create ` method. - :return: Remote reference""" + :return: :class:`Remote ` reference + """ return Remote.create(self, name, url, **kwargs) def delete_remote(self, remote: "Remote") -> str: @@ -531,8 +569,8 @@ def delete_remote(self, remote: "Remote") -> str: def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[PathLike] = None) -> str: if git_dir is None: git_dir = self.git_dir - # we do not support an absolute path of the gitconfig on windows , - # use the global config instead + # We do not support an absolute path of the gitconfig on Windows. + # Use the global config instead. if is_win and config_level == "system": config_level = "global" @@ -561,17 +599,20 @@ def config_reader( ) -> GitConfigParser: """ :return: - GitConfigParser allowing to read the full git configuration, but not to write it + :class:`GitConfigParser ` allowing to read the + full git configuration, but not to write it. The configuration will include values from the system, user and repository configuration files. :param config_level: - For possible values, see config_writer method - If None, all applicable levels will be used. Specify a level in case - you know which file you wish to read to prevent reading multiple files. - :note: On windows, system configuration cannot currently be read as the path is - unknown, instead the global path will be used.""" + For possible values, see the :meth:`config_writer` method. If None, all + applicable levels will be used. Specify a level in case you know which file + you wish to read to prevent reading multiple files. + + :note: On Windows, system configuration cannot currently be read as the path is + unknown, instead the global path will be used. + """ return self._config_reader(config_level=config_level) def _config_reader( @@ -592,23 +633,26 @@ def _config_reader( def config_writer(self, config_level: Lit_config_levels = "repository") -> GitConfigParser: """ :return: - GitConfigParser allowing to write values of the specified configuration file level. - Config writers should be retrieved, used to change the configuration, and written - right away as they will lock the configuration file in question and prevent other's - to write it. + A :class:`GitConfigParser ` allowing to write + values of the specified configuration file level. Config writers should be + retrieved, used to change the configuration, and written right away as they + will lock the configuration file in question and prevent other's to write + it. :param config_level: - One of the following values - system = system wide configuration file - global = user level configuration file - repository = configuration file for this repository only""" + One of the following values: + + * ``"system"`` = system wide configuration file + * ``"global"`` = user level configuration file + * ``"`repository"`` = configuration file for this repository only + """ return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self, merge_includes=False) def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: - """The Commit object for the specified revision + """The Commit object for the specified revision. :param rev: revision specifier, see git-rev-parse for viable options. - :return: ``git.Commit`` + :return: :class:`git.Commit ` """ if rev is None: return self.head.commit @@ -616,22 +660,27 @@ def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: def iter_trees(self, *args: Any, **kwargs: Any) -> Iterator["Tree"]: """:return: Iterator yielding Tree objects - :note: Takes all arguments known to iter_commits method""" + + :note: Accepts all arguments known to the :meth:`iter_commits` method. + """ return (c.tree for c in self.iter_commits(*args, **kwargs)) def tree(self, rev: Union[Tree_ish, str, None] = None) -> "Tree": - """The Tree object for the given treeish revision + """The Tree object for the given tree-ish revision. + Examples:: repo.tree(repo.heads[0]) - :param rev: is a revision pointing to a Treeish ( being a commit or tree ) + :param rev: is a revision pointing to a Treeish (being a commit or tree) + :return: ``git.Tree`` :note: If you need a non-root level tree, find it by iterating the root tree. Otherwise it cannot know about its path relative to the repository root and subsequent - operations might have unexpected results.""" + operations might have unexpected results. + """ if rev is None: return self.head.commit.tree return self.rev_parse(str(rev) + "^{tree}") @@ -642,37 +691,40 @@ def iter_commits( paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any, ) -> Iterator[Commit]: - """A list of Commit objects representing the history of a given ref/commit + """A list of Commit objects representing the history of a given ref/commit. :param rev: - revision specifier, see git-rev-parse for viable options. + Revision specifier, see git-rev-parse for viable options. If None, the active branch will be used. :param paths: - is an optional path or a list of paths; if set only commits that include the path - or paths will be returned + An optional path or a list of paths; if set only commits that include the + path or paths will be returned :param kwargs: - Arguments to be passed to git-rev-list - common ones are - max_count and skip + Arguments to be passed to git-rev-list - common ones are max_count and skip. - :note: to receive only commits between two named revisions, use the - "revA...revB" revision specifier + :note: To receive only commits between two named revisions, use the + ``"revA...revB"`` revision specifier. - :return: ``git.Commit[]``""" + :return: ``git.Commit[]`` + """ if rev is None: rev = self.head.commit return Commit.iter_items(self, rev, paths, **kwargs) def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: - """Find the closest common ancestor for the given revision (e.g. Commits, Tags, References, etc) + """Find the closest common ancestor for the given revision (Commits, Tags, References, etc.). :param rev: At least two revs to find the common ancestor for. - :param kwargs: Additional arguments to be passed to the repo.git.merge_base() command which does all the work. - :return: A list of Commit objects. If --all was not specified as kwarg, the list will have at max one Commit, - or is empty if no common merge base exists. - :raises ValueError: If not at least two revs are provided + :param kwargs: Additional arguments to be passed to the + ``repo.git.merge_base()`` command which does all the work. + :return: A list of :class:`Commit ` objects. If + ``--all`` was not passed as a keyword argument, the list will have at max + one :class:`Commit `, or is empty if no common + merge base exists. + :raises ValueError: If not at least two revs are provided. """ if len(rev) < 2: raise ValueError("Please specify at least two revs, got only %i" % len(rev)) @@ -697,7 +749,7 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: return res def is_ancestor(self, ancestor_rev: "Commit", rev: "Commit") -> bool: - """Check if a commit is an ancestor of another + """Check if a commit is an ancestor of another. :param ancestor_rev: Rev which should be an ancestor :param rev: Rev to test against ancestor_rev @@ -754,9 +806,10 @@ def _set_daemon_export(self, value: object) -> None: del _set_daemon_export def _get_alternates(self) -> List[str]: - """The list of alternates for this repo from which objects can be retrieved + """The list of alternates for this repo from which objects can be retrieved. - :return: list of strings being pathnames of alternates""" + :return: List of strings being pathnames of alternates + """ if self.git_dir: alternates_path = osp.join(self.git_dir, "objects", "info", "alternates") @@ -767,16 +820,18 @@ def _get_alternates(self) -> List[str]: return [] def _set_alternates(self, alts: List[str]) -> None: - """Sets the alternates + """Sets the alternates. :param alts: is the array of string paths representing the alternates at which git should look for objects, i.e. /home/user/repo/.git/objects :raise NoSuchPathError: + :note: The method does not check for the existence of the paths in alts - as the caller is responsible.""" + as the caller is responsible. + """ alternates_path = osp.join(self.common_dir, "objects", "info", "alternates") if not alts: if osp.isfile(alternates_path): @@ -801,27 +856,28 @@ def is_dirty( ) -> bool: """ :return: - ``True``, the repository is considered dirty. By default it will react + ``True`` if the repository is considered dirty. By default it will react like a git-status without untracked files, hence it is dirty if the - index or the working copy have changes.""" + index or the working copy have changes. + """ if self._bare: # Bare repositories with no associated working directory are # always considered to be clean. return False - # start from the one which is fastest to evaluate + # Start from the one which is fastest to evaluate. default_args = ["--abbrev=40", "--full-index", "--raw"] if not submodules: default_args.append("--ignore-submodules") if path: default_args.extend(["--", str(path)]) if index: - # diff index against HEAD + # diff index against HEAD. if osp.isfile(self.index.path) and len(self.git.diff("--cached", *default_args)): return True # END index handling if working_tree: - # diff index against working tree + # diff index against working tree. if len(self.git.diff(*default_args)): return True # END working tree handling @@ -841,14 +897,15 @@ def untracked_files(self) -> List[str]: are relative to the current working directory of the git command. :note: - ignored files will not appear here, i.e. files mentioned in .gitignore + Ignored files will not appear here, i.e. files mentioned in ``.gitignore``. :note: - This property is expensive, as no cache is involved. To process the result, please - consider caching it yourself.""" + This property is expensive, as no cache is involved. To process the result, + please consider caching it yourself. + """ return self._get_untracked_files() def _get_untracked_files(self, *args: Any, **kwargs: Any) -> List[str]: - # make sure we get all files, not only untracked directories + # Make sure we get all files, not only untracked directories. proc = self.git.status(*args, porcelain=True, untracked_files=True, as_process=True, **kwargs) # Untracked files prefix in porcelain mode prefix = "?? " @@ -868,11 +925,13 @@ def _get_untracked_files(self, *args: Any, **kwargs: Any) -> List[str]: return untracked_files def ignored(self, *paths: PathLike) -> List[str]: - """Checks if paths are ignored via .gitignore - Doing so using the "git check-ignore" method. + """Checks if paths are ignored via .gitignore. + + This does so using the ``git check-ignore`` method. :param paths: List of paths to check whether they are ignored or not - :return: subset of those paths which are ignored + + :return: Subset of those paths which are ignored """ try: proc: str = self.git.check_ignore(*paths) @@ -892,23 +951,25 @@ def active_branch(self) -> Head: """The name of the currently active branch. :raises TypeError: If HEAD is detached - :return: Head to the active branch""" + :return: Head to the active branch + """ # reveal_type(self.head.reference) # => Reference return self.head.reference def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterator["BlameEntry"]: """Iterator for blame information for the given file at the given revision. - Unlike .blame(), this does not return the actual file's contents, only - a stream of BlameEntry tuples. + Unlike :meth:`blame`, this does not return the actual file's contents, only a + stream of :class:`BlameEntry` tuples. - :param rev: revision specifier, see git-rev-parse for viable options. - :return: lazy iterator of BlameEntry tuples, where the commit - indicates the commit to blame for the line, and range - indicates a span of line numbers in the resulting file. + :param rev: Revision specifier, see git-rev-parse for viable options. - If you combine all line number ranges outputted by this command, you - should get a continuous range spanning all line numbers in the file. + :return: Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates + the commit to blame for the line, and range indicates a span of line numbers + in the resulting file. + + If you combine all line number ranges outputted by this command, you should get + a continuous range spanning all line numbers in the file. """ data: bytes = self.git.blame(rev, "--", file, p=True, incremental=True, stdout_as_string=False, **kwargs) @@ -917,7 +978,7 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat stream = (line for line in data.split(b"\n") if line) while True: try: - line = next(stream) # when exhausted, causes a StopIteration, terminating this function + line = next(stream) # When exhausted, causes a StopIteration, terminating this function. except StopIteration: return split_line = line.split() @@ -927,7 +988,7 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat orig_lineno = int(orig_lineno_b) if hexsha not in commits: # Now read the next few lines and build up a dict of properties - # for this commit + # for this commit. props: Dict[bytes, bytes] = {} while True: try: @@ -936,13 +997,13 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat return if line == b"boundary": # "boundary" indicates a root commit and occurs - # instead of the "previous" tag + # instead of the "previous" tag. continue tag, value = line.split(b" ", 1) props[tag] = value if tag == b"filename": - # "filename" formally terminates the entry for --incremental + # "filename" formally terminates the entry for --incremental. orig_filename = value break @@ -963,10 +1024,10 @@ def blame_incremental(self, rev: str | HEAD, file: str, **kwargs: Any) -> Iterat commits[hexsha] = c else: # Discard all lines until we find "filename" which is - # guaranteed to be the last line + # guaranteed to be the last line. while True: try: - line = next(stream) # will fail if we reach the EOF unexpectedly + line = next(stream) # Will fail if we reach the EOF unexpectedly. except StopIteration: return tag, value = line.split(b" ", 1) @@ -991,12 +1052,15 @@ def blame( ) -> List[List[Commit | List[str | bytes] | None]] | Iterator[BlameEntry] | None: """The blame information for the given file at the given revision. - :param rev: revision specifier, see git-rev-parse for viable options. + :param rev: Revision specifier, see git-rev-parse for viable options. + :return: list: [git.Commit, list: []] + A list of lists associating a Commit object with a list of lines that changed within the given commit. The Commit objects will be given in order - of appearance.""" + of appearance. + """ if incremental: return self.blame_incremental(rev, file, **kwargs) rev_opts = rev_opts or [] @@ -1028,9 +1092,9 @@ class InfoTD(TypedDict, total=False): is_binary = True else: # As we don't have an idea when the binary data ends, as it could contain multiple newlines - # in the process. So we rely on being able to decode to tell us what is is. + # in the process. So we rely on being able to decode to tell us what it is. # This can absolutely fail even on text files, but even if it does, we should be fine treating it - # as binary instead + # as binary instead. parts = self.re_whitespace.split(line_str, 1) firstpart = parts[0] is_binary = False @@ -1133,32 +1197,32 @@ def init( expand_vars: bool = True, **kwargs: Any, ) -> "Repo": - """Initialize a git repository at the given path if specified + """Initialize a git repository at the given path if specified. :param path: - is the full path to the repo (traditionally ends with /.git) - or None in which case the repository will be created in the current - working directory + The full path to the repo (traditionally ends with /.git) or None in + which case the repository will be created in the current working directory :param mkdir: - if specified will create the repository directory if it doesn't - already exists. Creates the directory with a mode=0755. - Only effective if a path is explicitly given + If specified, will create the repository directory if it doesn't + already exist. Creates the directory with a mode=0755. + Only effective if a path is explicitly given. :param odbt: Object DataBase type - a type which is constructed by providing the directory containing the database objects, i.e. .git/objects. - It will be used to access all object data + It will be used to access all object data. :param expand_vars: - if specified, environment variables will not be escaped. This + If specified, environment variables will not be escaped. This can lead to information disclosure, allowing attackers to - access the contents of environment variables + access the contents of environment variables. :param kwargs: - keyword arguments serving as additional options to the git-init command + Keyword arguments serving as additional options to the git-init command. - :return: ``git.Repo`` (the newly created repo)""" + :return: ``git.Repo`` (the newly created repo) + """ if path: path = expand_path(path, expand_vars) if mkdir and path and not osp.exists(path): @@ -1184,7 +1248,7 @@ def _clone( ) -> "Repo": odbt = kwargs.pop("odbt", odb_default_type) - # when pathlib.Path or other classbased path is passed + # When pathlib.Path or other classbased path is passed if not isinstance(path, str): path = str(path) @@ -1236,21 +1300,21 @@ def _clone( log.debug("Cmd(%s)'s unused stdout: %s", cmdline, stdout) finalize_process(proc, stderr=stderr) - # our git command could have a different working dir than our actual - # environment, hence we prepend its working dir if required + # Our git command could have a different working dir than our actual + # environment, hence we prepend its working dir if required. if not osp.isabs(path): path = osp.join(git._working_dir, path) if git._working_dir is not None else path repo = cls(path, odbt=odbt) - # retain env values that were passed to _clone() + # Retain env values that were passed to _clone(). repo.git.update_environment(**git.environment()) - # adjust remotes - there may be operating systems which use backslashes, + # Adjust remotes - there may be operating systems which use backslashes, # These might be given as initial paths, but when handling the config file # that contains the remote from which we were clones, git stops liking it # as it will escape the backslashes. Hence we undo the escaping just to be - # sure + # sure. if repo.remotes: with repo.remotes[0].config_writer as writer: writer.set_value("url", Git.polish_url(repo.remotes[0].url)) @@ -1268,20 +1332,22 @@ def clone( ) -> "Repo": """Create a clone from this repository. - :param path: is the full path of the new repo (traditionally ends with ./.git). - :param progress: See 'git.remote.Remote.push'. - :param multi_options: A list of Clone options that can be provided multiple times. One - option per list item which is passed exactly as specified to clone. - For example ['--config core.filemode=false', '--config core.ignorecase', + :param path: The full path of the new repo (traditionally ends with + ``./.git``). + :param progress: See :meth:`git.remote.Remote.push`. + :param multi_options: A list of Clone options that can be provided multiple times. + One option per list item which is passed exactly as specified to clone. + For example: ['--config core.filemode=false', '--config core.ignorecase', '--recurse-submodule=repo1_path', '--recurse-submodule=repo2_path'] - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack + :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. :param kwargs: * odbt = ObjectDatabase Type, allowing to determine the object database - implementation used by the returned Repo instance - * All remaining keyword arguments are given to the git-clone command + implementation used by the returned Repo instance. + * All remaining keyword arguments are given to the git-clone command. - :return: ``git.Repo`` (the newly cloned repo)""" + :return: :class:`Repo` (the newly cloned repo) + """ return self._clone( self.git, self.common_dir, @@ -1306,22 +1372,32 @@ def clone_from( allow_unsafe_options: bool = False, **kwargs: Any, ) -> "Repo": - """Create a clone from the given URL + """Create a clone from the given URL. + + :param url: Valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS + + :param to_path: Path to which the repository should be cloned to. + + :param progress: See :meth:`git.remote.Remote.push`. - :param url: valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS - :param to_path: Path to which the repository should be cloned to - :param progress: See 'git.remote.Remote.push'. :param env: Optional dictionary containing the desired environment variables. + Note: Provided variables will be used to update the execution environment for `git`. If some variable is not specified in `env` and is defined in `os.environ`, value from `os.environ` will be used. If you want to unset some variable, consider providing empty string as its value. - :param multi_options: See ``clone`` method - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: see the ``clone`` method - :return: Repo instance pointing to the cloned directory""" + + :param multi_options: See :meth:`clone` method. + + :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + + :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + + :param kwargs: See the :meth:`clone` method. + + :return: :class:`Repo` instance pointing to the cloned directory. + """ git = cls.GitCommandWrapperType(os.getcwd()) if env is not None: git.update_environment(**env) @@ -1346,18 +1422,23 @@ def archive( ) -> Repo: """Archive the tree at the given revision. - :param ostream: file compatible stream object to which the archive will be written as bytes - :param treeish: is the treeish name/id, defaults to active branch - :param prefix: is the optional prefix to prepend to each filename in the archive - :param kwargs: Additional arguments passed to git-archive + :param ostream: file compatible stream object to which the archive will be written as bytes. + + :param treeish: is the treeish name/id, defaults to active branch. + + :param prefix: is the optional prefix to prepend to each filename in the archive. + + :param kwargs: Additional arguments passed to git-archive: * Use the 'format' argument to define the kind of format. Use specialized ostreams to write any format supported by python. * You may specify the special **path** keyword, which may either be a repository-relative path to a directory or file to place into the archive, or a list or tuple of multiple paths. - :raise GitCommandError: in case something went wrong - :return: self""" + :raise GitCommandError: If something went wrong. + + :return: self + """ if treeish is None: treeish = self.head.commit if prefix and "prefix" not in kwargs: @@ -1374,7 +1455,8 @@ def archive( def has_separate_working_tree(self) -> bool: """ :return: True if our git_dir is not at the root of our working_tree_dir, but a .git file with a - platform agnositic symbolic link. Our git_dir will be wherever the .git file points to + platform agnositic symbolic link. Our git_dir will be wherever the .git file points to. + :note: bare repositories will always return False here """ if self.bare: @@ -1382,7 +1464,7 @@ def has_separate_working_tree(self) -> bool: if self.working_tree_dir: return osp.isfile(osp.join(self.working_tree_dir, ".git")) else: - return False # or raise Error? + return False # Or raise Error? rev_parse = rev_parse @@ -1394,7 +1476,7 @@ def currently_rebasing_on(self) -> Commit | None: """ :return: The commit which is currently being replayed while rebasing. - None if we are not currently rebasing. + None if we are not currently rebasing. """ if self.git_dir: rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") diff --git a/git/repo/fun.py b/git/repo/fun.py index ae35aa81e..f0bb4cd1f 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -1,5 +1,7 @@ -"""Package with general repository related functions""" +"""Module with general repository-related functions.""" + from __future__ import annotations + import os import stat from pathlib import Path diff --git a/git/types.py b/git/types.py index 21276b5f1..22bb91c16 100644 --- a/git/types.py +++ b/git/types.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + # flake8: noqa import os @@ -75,9 +76,11 @@ def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, None] = None) -> None: """For use in exhaustive checking of literal or Enum in if/else chain. + Should only be reached if all members not handled OR attempt to pass non-members through chain. If all members handled, type is Empty. Otherwise, will cause mypy error. + If non-members given, should cause mypy error at variable creation. If raise_error is True, will also raise AssertionError or the Exception passed to exc. diff --git a/git/util.py b/git/util.py index ccadb5881..d6a20b3e3 100644 --- a/git/util.py +++ b/git/util.py @@ -144,7 +144,7 @@ def _read_env_flag(name: str, default: bool) -> bool: def unbare_repo(func: Callable[..., T]) -> Callable[..., T]: """Methods with this decorator raise :class:`.exc.InvalidGitRepositoryError` if they - encounter a bare repository""" + encounter a bare repository.""" from .exc import InvalidGitRepositoryError @@ -164,8 +164,9 @@ def wrapper(self: "Remote", *args: Any, **kwargs: Any) -> T: def cwd(new_dir: PathLike) -> Generator[PathLike, None, None]: """Context manager to temporarily change directory. - This is similar to contextlib.chdir introduced in Python 3.11, but the context - manager object returned by a single call to this function is not reentrant.""" + This is similar to :func:`contextlib.chdir` introduced in Python 3.11, but the + context manager object returned by a single call to this function is not reentrant. + """ old_dir = os.getcwd() os.chdir(new_dir) try: @@ -191,8 +192,10 @@ def patch_env(name: str, value: str) -> Generator[None, None, None]: def rmtree(path: PathLike) -> None: """Remove the given directory tree recursively. - :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files that - couldn't be deleted are read-only. Windows will not remove them in that case.""" + :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files + that couldn't be deleted are read-only. Windows will not remove them in that + case. + """ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: """Callback for :func:`shutil.rmtree`. Works either as ``onexc`` or ``onerror``.""" @@ -224,9 +227,10 @@ def rmfile(path: PathLike) -> None: def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * 1024) -> int: """Copy all data from the source stream into the destination stream in chunks - of size chunk_size + of size chunk_size. - :return: amount of bytes written""" + :return: Number of bytes written + """ br = 0 while True: chunk = source.read(chunk_size) @@ -240,7 +244,7 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: R"""Join path tokens together similar to osp.join, but always use - '/' instead of possibly '\' on windows.""" + '/' instead of possibly '\' on Windows.""" path = str(a) for b in p: b = str(b) @@ -269,7 +273,7 @@ def to_native_path_linux(path: PathLike) -> str: __all__.append("to_native_path_windows") to_native_path = to_native_path_windows else: - # no need for any work on linux + # No need for any work on Linux. def to_native_path_linux(path: PathLike) -> str: return str(path) @@ -277,19 +281,22 @@ def to_native_path_linux(path: PathLike) -> str: def join_path_native(a: PathLike, *p: PathLike) -> PathLike: - R""" - As join path, but makes sure an OS native path is returned. This is only - needed to play it safe on my dear windows and to assure nice paths that only - use '\'""" + R"""Like join_path, but makes sure an OS native path is returned. + + This is only needed to play it safe on Windows and to assure nice paths that only + use '\'. + """ return to_native_path(join_path(a, *p)) def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: - """Assure that the directory pointed to by path exists. + """Make sure that the directory pointed to by path exists. + + :param is_file: If True, ``path`` is assumed to be a file and handled correctly. + Otherwise it must be a directory. - :param is_file: If True, path is assumed to be a file and handled correctly. - Otherwise it must be a directory - :return: True if the directory was created, False if it already existed""" + :return: True if the directory was created, False if it already existed. + """ if is_file: path = osp.dirname(path) # END handle file @@ -365,9 +372,9 @@ def _cygexpath(drive: Optional[str], path: str) -> str: def cygpath(path: str) -> str: - """Use :meth:`git.cmd.Git.polish_url()` instead, that works on any environment.""" - path = str(path) # ensure is str and not AnyPath. - # Fix to use Paths when 3.5 dropped. or to be just str if only for urls? + """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" + path = str(path) # Ensure is str and not AnyPath. + # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? if not path.startswith(("/cygdrive", "//", "/proc/cygdrive")): for regex, parser, recurse in _cygpath_parsers: match = regex.match(path) @@ -477,11 +484,10 @@ def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: - """ - Parse any command line argument and if on of the element is an URL with a + """Parse any command line argument and if one of the elements is an URL with a username and/or password, replace them by stars (in-place). - If nothing found just returns the command line as-is. + If nothing is found, this just returns the command line as-is. This should be used for every log line that print a command line, as well as exception messages. @@ -491,7 +497,7 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: new_cmdline.append(to_parse) try: url = urlsplit(to_parse) - # Remove password from the URL if present + # Remove password from the URL if present. if url.password is None and url.username is None: continue @@ -501,7 +507,7 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: url = url._replace(netloc=url.netloc.replace(url.username, "*****")) new_cmdline[index] = urlunsplit(url) except ValueError: - # This is not a valid URL + # This is not a valid URL. continue return new_cmdline @@ -555,14 +561,15 @@ def _parse_progress_line(self, line: AnyStr) -> None: or git-fetch. - Lines that do not contain progress info are stored in :attr:`other_lines`. - - Lines that seem to contain an error (i.e. start with error: or fatal:) are stored - in :attr:`error_lines`.""" + - Lines that seem to contain an error (i.e. start with ``error:`` or ``fatal:``) + are stored in :attr:`error_lines`. + """ # handle # Counting objects: 4, done. # Compressing objects: 50% (1/2) # Compressing objects: 100% (2/2) # Compressing objects: 100% (2/2), done. - if isinstance(line, bytes): # mypy argues about ternary assignment + if isinstance(line, bytes): # mypy argues about ternary assignment. line_str = line.decode("utf-8") else: line_str = line @@ -572,14 +579,14 @@ def _parse_progress_line(self, line: AnyStr) -> None: self.error_lines.append(self._cur_line) return - # find escape characters and cut them away - regex will not work with - # them as they are non-ascii. As git might expect a tty, it will send them + # Find escape characters and cut them away - regex will not work with + # them as they are non-ASCII. As git might expect a tty, it will send them. last_valid_index = None for i, c in enumerate(reversed(line_str)): if ord(c) < 32: # its a slice index last_valid_index = -i - 1 - # END character was non-ascii + # END character was non-ASCII # END for each character in line if last_valid_index is not None: line_str = line_str[:last_valid_index] @@ -600,7 +607,7 @@ def _parse_progress_line(self, line: AnyStr) -> None: op_code = 0 _remote, op_name, _percent, cur_count, max_count, message = match.groups() - # get operation id + # Get operation ID. if op_name == "Counting objects": op_code |= self.COUNTING elif op_name == "Compressing objects": @@ -616,7 +623,7 @@ def _parse_progress_line(self, line: AnyStr) -> None: elif op_name == "Checking out files": op_code |= self.CHECKING_OUT else: - # Note: On windows it can happen that partial lines are sent + # Note: On Windows it can happen that partial lines are sent. # Hence we get something like "CompreReceiving objects", which is # a blend of "Compressing objects" and "Receiving objects". # This can't really be prevented, so we drop the line verbosely @@ -624,11 +631,11 @@ def _parse_progress_line(self, line: AnyStr) -> None: # commands at some point. self.line_dropped(line_str) # Note: Don't add this line to the other lines, as we have to silently - # drop it + # drop it. return None # END handle op code - # figure out stage + # Figure out stage. if op_code not in self._seen_ops: self._seen_ops.append(op_code) op_code |= self.BEGIN @@ -655,8 +662,9 @@ def _parse_progress_line(self, line: AnyStr) -> None: def new_message_handler(self) -> Callable[[str], None]: """ :return: - a progress handler suitable for handle_process_output(), passing lines on to this Progress - handler in a suitable format""" + A progress handler suitable for handle_process_output(), passing lines on to + this Progress handler in a suitable format + """ def handler(line: AnyStr) -> None: return self._parse_progress_line(line.rstrip()) @@ -675,7 +683,7 @@ def update( max_count: Union[str, float, None] = None, message: str = "", ) -> None: - """Called whenever the progress changes + """Called whenever the progress changes. :param op_code: Integer allowing to be compared against Operation IDs and stage IDs. @@ -683,11 +691,12 @@ def update( Stage IDs are BEGIN and END. BEGIN will only be set once for each Operation ID as well as END. It may be that BEGIN and END are set at once in case only one progress message was emitted due to the speed of the operation. - Between BEGIN and END, none of these flags will be set + Between BEGIN and END, none of these flags will be set. Operation IDs are all held within the OP_MASK. Only one Operation ID will be active per call. - :param cur_count: Current absolute count of items + + :param cur_count: Current absolute count of items. :param max_count: The maximum count of items we expect. It may be None in case there is @@ -697,12 +706,13 @@ def update( In case of the 'WRITING' operation, it contains the amount of bytes transferred. It may possibly be used for other purposes as well. - You may read the contents of the current line in self._cur_line""" + You may read the contents of the current line in ``self._cur_line``. + """ pass class CallableRemoteProgress(RemoteProgress): - """An implementation forwarding updates to any callable""" + """An implementation forwarding updates to any callable.""" __slots__ = "_callable" @@ -724,7 +734,7 @@ class Actor(object): name_email_regex = re.compile(r"(.*) <(.*?)>") # ENVIRONMENT VARIABLES - # read when creating new commits + # These are read when creating new commits. env_author_name = "GIT_AUTHOR_NAME" env_author_email = "GIT_AUTHOR_EMAIL" env_committer_name = "GIT_COMMITTER_NAME" @@ -758,11 +768,13 @@ def __repr__(self) -> str: @classmethod def _from_string(cls, string: str) -> "Actor": """Create an Actor from a string. - :param string: is the string, which is expected to be in regular git format - John Doe + :param string: The string, which is expected to be in regular git format:: - :return: Actor""" + John Doe + + :return: Actor + """ m = cls.name_email_regex.search(string) if m: name, email = m.groups() @@ -771,7 +783,7 @@ def _from_string(cls, string: str) -> "Actor": m = cls.name_only_regex.search(string) if m: return Actor(m.group(1), None) - # assume best and use the whole string as name + # Assume the best and use the whole string as name. return Actor(string, None) # END special case name # END handle name/email matching @@ -784,7 +796,7 @@ def _main_actor( config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None, ) -> "Actor": actor = Actor("", "") - user_id = None # We use this to avoid multiple calls to getpass.getuser() + user_id = None # We use this to avoid multiple calls to getpass.getuser(). def default_email() -> str: nonlocal user_id @@ -822,20 +834,21 @@ def committer(cls, config_reader: Union[None, "GitConfigParser", "SectionConstra :return: Actor instance corresponding to the configured committer. It behaves similar to the git implementation, such that the environment will override configuration values of config_reader. If no value is set at all, it will be - generated + generated. + :param config_reader: ConfigReader to use to retrieve the values from in case - they are not set in the environment""" + they are not set in the environment. + """ return cls._main_actor(cls.env_committer_name, cls.env_committer_email, config_reader) @classmethod def author(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": - """Same as committer(), but defines the main author. It may be specified in the environment, - but defaults to the committer""" + """Same as committer(), but defines the main author. It may be specified in the + environment, but defaults to the committer.""" return cls._main_actor(cls.env_author_name, cls.env_author_email, config_reader) class Stats(object): - """ Represents stat information as presented by git at the end of a merge. It is created from the output of a diff operation. @@ -859,7 +872,8 @@ class Stats(object): In addition to the items in the stat-dict, it features additional information:: - files = number of changed files as int""" + files = number of changed files as int + """ __slots__ = ("total", "files") @@ -871,7 +885,8 @@ def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]): def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": """Create a Stat object from output retrieved by git-diff. - :return: git.Stat""" + :return: git.Stat + """ hsh: HSH_TD = { "total": {"insertions": 0, "deletions": 0, "lines": 0, "files": 0}, @@ -895,14 +910,14 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": class IndexFileSHA1Writer(object): - """Wrapper around a file-like object that remembers the SHA1 of the data written to it. It will write a sha when the stream is closed or if the asked for explicitly using write_sha. - Only useful to the indexfile + Only useful to the index file. - :note: Based on the dulwich project""" + :note: Based on the dulwich project. + """ __slots__ = ("f", "sha1") @@ -929,13 +944,13 @@ def tell(self) -> int: class LockFile(object): - """Provides methods to obtain, check for, and release a file based lock which should be used to handle concurrent access to the same file. As we are a utility class to be derived from, we only use protected methods. - Locks will automatically be released on destruction""" + Locks will automatically be released on destruction. + """ __slots__ = ("_file_path", "_owns_lock") @@ -951,14 +966,18 @@ def _lock_file_path(self) -> str: return "%s.lock" % (self._file_path) def _has_lock(self) -> bool: - """:return: True if we have a lock and if the lockfile still exists - :raise AssertionError: if our lock-file does not exist""" + """ + :return: True if we have a lock and if the lockfile still exists + + :raise AssertionError: If our lock-file does not exist + """ return self._owns_lock def _obtain_lock_or_raise(self) -> None: - """Create a lock file as flag for other instances, mark our instance as lock-holder + """Create a lock file as flag for other instances, mark our instance as lock-holder. - :raise IOError: if a lock was already present or a lock file could not be written""" + :raise IOError: If a lock was already present or a lock file could not be written + """ if self._has_lock(): return lock_file = self._lock_file_path() @@ -978,15 +997,15 @@ def _obtain_lock_or_raise(self) -> None: def _obtain_lock(self) -> None: """The default implementation will raise if a lock cannot be obtained. - Subclasses may override this method to provide a different implementation""" + Subclasses may override this method to provide a different implementation.""" return self._obtain_lock_or_raise() def _release_lock(self) -> None: - """Release our lock if we have one""" + """Release our lock if we have one.""" if not self._has_lock(): return - # if someone removed our file beforhand, lets just flag this issue + # If someone removed our file beforehand, lets just flag this issue # instead of failing, to make it more usable. lfp = self._lock_file_path() try: @@ -997,13 +1016,13 @@ def _release_lock(self) -> None: class BlockingLockFile(LockFile): - """The lock file will block until a lock could be obtained, or fail after a specified timeout. :note: If the directory containing the lock was removed, an exception will be raised during the blocking period, preventing hangs as the lock - can never be obtained.""" + can never be obtained. + """ __slots__ = ("_check_interval", "_max_block_time") @@ -1013,13 +1032,14 @@ def __init__( check_interval_s: float = 0.3, max_block_time_s: int = sys.maxsize, ) -> None: - """Configure the instance + """Configure the instance. :param check_interval_s: Period of time to sleep until the lock is checked the next time. - By default, it waits a nearly unlimited time + By default, it waits a nearly unlimited time. - :param max_block_time_s: Maximum amount of seconds we may lock""" + :param max_block_time_s: Maximum amount of seconds we may lock. + """ super(BlockingLockFile, self).__init__(file_path) self._check_interval = check_interval_s self._max_block_time = max_block_time_s @@ -1027,7 +1047,9 @@ def __init__( def _obtain_lock(self) -> None: """This method blocks until it obtained the lock, or raises IOError if it ran out of time or if the parent directory was not available anymore. - If this method returns, you are guaranteed to own the lock""" + + If this method returns, you are guaranteed to own the lock. + """ starttime = time.time() maxtime = starttime + float(self._max_block_time) while True: @@ -1059,7 +1081,6 @@ def _obtain_lock(self) -> None: class IterableList(List[T_IterableObj]): - """ List of iterable objects allowing to query an object by id or by named index:: @@ -1070,13 +1091,14 @@ class IterableList(List[T_IterableObj]): Iterable parent objects = [Commit, SubModule, Reference, FetchInfo, PushInfo] Iterable via inheritance = [Head, TagReference, RemoteReference] - ] + It requires an id_attribute name to be set which will be queried from its contained items to have a means for comparison. A prefix can be specified which is to be used in case the id returned by the items always contains a prefix that does not matter to the user, so it - can be left out.""" + can be left out. + """ __slots__ = ("_id_attr", "_prefix") @@ -1088,7 +1110,7 @@ def __init__(self, id_attr: str, prefix: str = "") -> None: self._prefix = prefix def __contains__(self, attr: object) -> bool: - # first try identity match for performance + # First try identity match for performance. try: rval = list.__contains__(self, attr) if rval: @@ -1097,9 +1119,9 @@ def __contains__(self, attr: object) -> bool: pass # END handle match - # otherwise make a full name search + # Otherwise make a full name search. try: - getattr(self, cast(str, attr)) # use cast to silence mypy + getattr(self, cast(str, attr)) # Use cast to silence mypy. return True except (AttributeError, TypeError): return False @@ -1148,7 +1170,7 @@ def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: class IterableClassWatcher(type): - """Metaclass that watches""" + """Metaclass that watches.""" def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: for base in bases: @@ -1164,24 +1186,26 @@ def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: class Iterable(metaclass=IterableClassWatcher): - """Defines an interface for iterable items which is to assure a uniform - way to retrieve and iterate items within the git repository""" + way to retrieve and iterate items within the git repository.""" __slots__ = () + _id_attribute_ = "attribute that most suitably identifies your instance" @classmethod def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: """ Deprecated, use IterableObj instead. + Find all items of this type - subclasses can specify args and kwargs differently. If no args are given, subclasses are obliged to return all items if no additional arguments arg given. :note: Favor the iter_items method as it will - :return: list(Item,...) list of item instances""" + :return: list(Item,...) list of item instances + """ out_list: Any = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list @@ -1189,19 +1213,23 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: @classmethod def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: # return typed to be compatible with subtypes e.g. Remote - """For more information about the arguments, see list_items - :return: iterator yielding Items""" + """For more information about the arguments, see list_items. + + :return: Iterator yielding Items + """ raise NotImplementedError("To be implemented by Subclass") @runtime_checkable class IterableObj(Protocol): """Defines an interface for iterable items which is to assure a uniform - way to retrieve and iterate items within the git repository + way to retrieve and iterate items within the git repository. - Subclasses = [Submodule, Commit, Reference, PushInfo, FetchInfo, Remote]""" + Subclasses = [Submodule, Commit, Reference, PushInfo, FetchInfo, Remote] + """ __slots__ = () + _id_attribute_: str @classmethod @@ -1213,7 +1241,8 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I :note: Favor the iter_items method as it will - :return: list(Item,...) list of item instances""" + :return: list(Item,...) list of item instances + """ out_list: IterableList = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list @@ -1221,9 +1250,11 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I @classmethod @abstractmethod def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_IterableObj]: # Iterator[T_IterableObj]: - # return typed to be compatible with subtypes e.g. Remote - """For more information about the arguments, see list_items - :return: iterator yielding Items""" + # Return-typed to be compatible with subtypes e.g. Remote. + """For more information about the arguments, see list_items. + + :return: Iterator yielding Items + """ raise NotImplementedError("To be implemented by Subclass") From e8343e26f0d03fcdc3492b1412af7156cf17b636 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 11:48:44 -0400 Subject: [PATCH 0809/1790] Firm up comment about is_win in util.is_cygwin_git The is_cygwin_git function returns False when we have is_win, because is_win is not True on Cygwin systems. The existing comment explained why, but was tentative. The claims are accurate, so this rewrites the comment to state it more definitively. --- git/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/git/util.py b/git/util.py index d6a20b3e3..29cf12f7c 100644 --- a/git/util.py +++ b/git/util.py @@ -419,8 +419,7 @@ def is_cygwin_git(git_executable: PathLike) -> bool: def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: if is_win: - # is_win seems to be true only for Windows-native pythons - # cygwin has os.name = posix, I think + # is_win is only True on native Windows systems. On Cygwin, os.name == "posix". return False if git_executable is None: From f78587fd9376d3578fa816e8c99f5d8dd657332c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 11:57:50 -0400 Subject: [PATCH 0810/1790] Remove explicit inheritance from object Within the git module. In Python 2, it was necessary to inherit from object to have a new-style class. In Python 3, all classes are new-style classes, and inheritance from object is implied. Usually, removing explicit inheritance from object is only a small clarity benefit while modernizing Python codebases. However, in GitPython, the benefit is greater, because it is possible to confuse object (the root of the Python class hierarchy) with Object (the root of the GitPython subhierarchy of classes representing things in the git object model). --- git/cmd.py | 4 ++-- git/diff.py | 6 +++--- git/index/typ.py | 2 +- git/index/util.py | 2 +- git/objects/tree.py | 2 +- git/objects/util.py | 2 +- git/refs/symbolic.py | 2 +- git/repo/base.py | 2 +- git/util.py | 10 +++++----- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index f8b6c9e78..384a39077 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -490,7 +490,7 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." ) - class AutoInterrupt(object): + class AutoInterrupt: """Process wrapper that terminates the wrapped process on finalization. This kills/interrupts the stored process instance once this instance goes out of @@ -602,7 +602,7 @@ def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> byte # END auto interrupt - class CatFileContentStream(object): + class CatFileContentStream: """Object representing a sized read-only stream returning the contents of an object. diff --git a/git/diff.py b/git/diff.py index 1fde4d676..d5740a7fc 100644 --- a/git/diff.py +++ b/git/diff.py @@ -79,7 +79,7 @@ def decode_path(path: bytes, has_ab_prefix: bool = True) -> Optional[bytes]: return path -class Diffable(object): +class Diffable: """Common interface for all objects that can be diffed against another object of compatible type. @@ -90,7 +90,7 @@ class Diffable(object): __slots__ = () - class Index(object): + class Index: """Stand-in indicating you want to diff against the index.""" def _process_diff_args( @@ -252,7 +252,7 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: # END for each diff -class Diff(object): +class Diff: """A Diff contains diff information between two Trees. It contains two sides a and b of the diff, members are prefixed with diff --git a/git/index/typ.py b/git/index/typ.py index 046df6e83..7b86cae55 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -32,7 +32,7 @@ # } END invariants -class BlobFilter(object): +class BlobFilter: """ Predicate to be used by iter_blobs allowing to filter only return blobs which match the given list of directories or files. diff --git a/git/index/util.py b/git/index/util.py index f52b61b4a..08e49d860 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -33,7 +33,7 @@ # } END aliases -class TemporaryFileSwap(object): +class TemporaryFileSwap: """Utility class moving a file to a temporary location within the same directory and moving it back on to where on object deletion.""" diff --git a/git/objects/tree.py b/git/objects/tree.py index fec98d6e8..b33da3709 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -102,7 +102,7 @@ def merge_sort(a: List[TreeCacheTup], cmp: Callable[[TreeCacheTup, TreeCacheTup] k = k + 1 -class TreeModifier(object): +class TreeModifier: """A utility class providing methods to alter the underlying cache in a list-like fashion. Once all adjustments are complete, the _cache, which really is a reference to diff --git a/git/objects/util.py b/git/objects/util.py index d2c1c0158..5ccb3ec37 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -337,7 +337,7 @@ def parse_actor_and_date(line: str) -> Tuple[Actor, int, int]: # { Classes -class ProcessStreamAdapter(object): +class ProcessStreamAdapter: """Class wiring all calls to the contained Process instance. Use this type to hide the underlying process to provide access only to a specified diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 7da957e2f..3587a52d3 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -55,7 +55,7 @@ def _git_dir(repo: "Repo", path: Union[PathLike, None]) -> PathLike: return repo.common_dir -class SymbolicReference(object): +class SymbolicReference: """Special case of a reference that is symbolic. This does not point to a specific commit, but to another diff --git a/git/repo/base.py b/git/repo/base.py index da81698d8..2694282d0 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -108,7 +108,7 @@ class BlameEntry(NamedTuple): orig_linenos: range -class Repo(object): +class Repo: """Represents a git repository and allows you to query references, gather commit information, generate diffs, create and clone repositories query the log. diff --git a/git/util.py b/git/util.py index 29cf12f7c..e61e3c6f5 100644 --- a/git/util.py +++ b/git/util.py @@ -516,7 +516,7 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: # { Classes -class RemoteProgress(object): +class RemoteProgress: """ Handler providing an interface to parse progress information emitted by git-push and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. @@ -723,7 +723,7 @@ def update(self, *args: Any, **kwargs: Any) -> None: self._callable(*args, **kwargs) -class Actor(object): +class Actor: """Actors hold information about a person acting on the repository. They can be committers and authors or anything with a name and an email as mentioned in the git log entries.""" @@ -847,7 +847,7 @@ def author(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint return cls._main_actor(cls.env_author_name, cls.env_author_email, config_reader) -class Stats(object): +class Stats: """ Represents stat information as presented by git at the end of a merge. It is created from the output of a diff operation. @@ -908,7 +908,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": return Stats(hsh["total"], hsh["files"]) -class IndexFileSHA1Writer(object): +class IndexFileSHA1Writer: """Wrapper around a file-like object that remembers the SHA1 of the data written to it. It will write a sha when the stream is closed or if the asked for explicitly using write_sha. @@ -942,7 +942,7 @@ def tell(self) -> int: return self.f.tell() -class LockFile(object): +class LockFile: """Provides methods to obtain, check for, and release a file based lock which should be used to handle concurrent access to the same file. From 0327f8f8cb07bc09070468fbf5b242fbc63a1589 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 18 Oct 2023 12:18:33 -0400 Subject: [PATCH 0811/1790] Make all one-element __slots__ be tuples Most already were, but a few were strings. --- git/index/typ.py | 2 +- git/objects/tree.py | 5 +++-- git/util.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/git/index/typ.py b/git/index/typ.py index 7b86cae55..9f57f067b 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -40,7 +40,7 @@ class BlobFilter: The given paths are given relative to the repository. """ - __slots__ = "paths" + __slots__ = ("paths",) def __init__(self, paths: Sequence[PathLike]) -> None: """ diff --git a/git/objects/tree.py b/git/objects/tree.py index b33da3709..c91c01807 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -109,7 +109,7 @@ class TreeModifier: the cache of a tree, will be sorted. This ensures it will be in a serializable state. """ - __slots__ = "_cache" + __slots__ = ("_cache",) def __init__(self, cache: List[TreeCacheTup]) -> None: self._cache = cache @@ -214,7 +214,8 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): """ type: Literal["tree"] = "tree" - __slots__ = "_cache" + + __slots__ = ("_cache",) # Actual integer IDs for comparison. commit_id = 0o16 # Equals stat.S_IFDIR | stat.S_IFLNK - a directory link. diff --git a/git/util.py b/git/util.py index e61e3c6f5..4e96f7394 100644 --- a/git/util.py +++ b/git/util.py @@ -713,7 +713,7 @@ def update( class CallableRemoteProgress(RemoteProgress): """An implementation forwarding updates to any callable.""" - __slots__ = "_callable" + __slots__ = ("_callable",) def __init__(self, fn: Callable) -> None: self._callable = fn From d4a87c1ad48faa23b0565111e1f0c14bbdfe10a6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 22 Oct 2023 02:55:35 -0400 Subject: [PATCH 0812/1790] Revise comments in tests used to generate tutorials --- test/test_docs.py | 267 +++++++++++++++++++++-------------------- test/test_quick_doc.py | 36 +++--- 2 files changed, 152 insertions(+), 151 deletions(-) diff --git a/test/test_docs.py b/test/test_docs.py index d1ed46926..1ca07f886 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -4,6 +4,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import os import sys @@ -34,7 +35,7 @@ def test_init_repo_object(self, rw_dir): # rorepo is a Repo instance pointing to the git-python repository. # For all you know, the first argument to Repo is a path to the repository - # you want to work with + # you want to work with. repo = Repo(self.rorepo.working_tree_dir) assert not repo.bare # ![1-test_init_repo_object] @@ -45,20 +46,20 @@ def test_init_repo_object(self, rw_dir): # ![2-test_init_repo_object] # [3-test_init_repo_object] - repo.config_reader() # get a config reader for read-only access - with repo.config_writer(): # get a config writer to change configuration - pass # call release() to be sure changes are written and locks are released + repo.config_reader() # Get a config reader for read-only access. + with repo.config_writer(): # Get a config writer to change configuration. + pass # Call release() to be sure changes are written and locks are released. # ![3-test_init_repo_object] # [4-test_init_repo_object] - assert not bare_repo.is_dirty() # check the dirty state - repo.untracked_files # retrieve a list of untracked files + assert not bare_repo.is_dirty() # Check the dirty state. + repo.untracked_files # Retrieve a list of untracked files. # ['my_untracked_file'] # ![4-test_init_repo_object] # [5-test_init_repo_object] cloned_repo = repo.clone(os.path.join(rw_dir, "to/this/path")) - assert cloned_repo.__class__ is Repo # clone an existing repository + assert cloned_repo.__class__ is Repo # Clone an existing repository. assert Repo.init(os.path.join(rw_dir, "path/for/new/repo")).__class__ is Repo # ![5-test_init_repo_object] @@ -69,9 +70,9 @@ def test_init_repo_object(self, rw_dir): # repository paths # [7-test_init_repo_object] - assert os.path.isdir(cloned_repo.working_tree_dir) # directory with your work files - assert cloned_repo.git_dir.startswith(cloned_repo.working_tree_dir) # directory containing the git repository - assert bare_repo.working_tree_dir is None # bare repositories have no working tree + assert os.path.isdir(cloned_repo.working_tree_dir) # Directory with your work files. + assert cloned_repo.git_dir.startswith(cloned_repo.working_tree_dir) # Directory containing the git repository. + assert bare_repo.working_tree_dir is None # Bare repositories have no working tree. # ![7-test_init_repo_object] # heads, tags and references @@ -79,59 +80,59 @@ def test_init_repo_object(self, rw_dir): # [8-test_init_repo_object] self.assertEqual( repo.head.ref, - repo.heads.master, # head is a sym-ref pointing to master + repo.heads.master, # head is a sym-ref pointing to master. "It's ok if TC not running from `master`.", ) - self.assertEqual(repo.tags["0.3.5"], repo.tag("refs/tags/0.3.5")) # you can access tags in various ways too - self.assertEqual(repo.refs.master, repo.heads["master"]) # .refs provides all refs, ie heads ... + self.assertEqual(repo.tags["0.3.5"], repo.tag("refs/tags/0.3.5")) # You can access tags in various ways too. + self.assertEqual(repo.refs.master, repo.heads["master"]) # .refs provides all refs, i.e. heads... if "TRAVIS" not in os.environ: self.assertEqual(repo.refs["origin/master"], repo.remotes.origin.refs.master) # ... remotes ... - self.assertEqual(repo.refs["0.3.5"], repo.tags["0.3.5"]) # ... and tags + self.assertEqual(repo.refs["0.3.5"], repo.tags["0.3.5"]) # ... and tags. # ![8-test_init_repo_object] - # create a new head/branch + # Create a new head/branch. # [9-test_init_repo_object] - new_branch = cloned_repo.create_head("feature") # create a new branch ... + new_branch = cloned_repo.create_head("feature") # Create a new branch ... assert cloned_repo.active_branch != new_branch # which wasn't checked out yet ... - self.assertEqual(new_branch.commit, cloned_repo.active_branch.commit) # pointing to the checked-out commit - # It's easy to let a branch point to the previous commit, without affecting anything else - # Each reference provides access to the git object it points to, usually commits + self.assertEqual(new_branch.commit, cloned_repo.active_branch.commit) # pointing to the checked-out commit. + # It's easy to let a branch point to the previous commit, without affecting anything else. + # Each reference provides access to the git object it points to, usually commits. assert new_branch.set_commit("HEAD~1").commit == cloned_repo.active_branch.commit.parents[0] # ![9-test_init_repo_object] - # create a new tag reference + # Create a new tag reference. # [10-test_init_repo_object] past = cloned_repo.create_tag( "past", ref=new_branch, message="This is a tag-object pointing to %s" % new_branch.name, ) - self.assertEqual(past.commit, new_branch.commit) # the tag points to the specified commit - assert past.tag.message.startswith("This is") # and its object carries the message provided + self.assertEqual(past.commit, new_branch.commit) # The tag points to the specified commit + assert past.tag.message.startswith("This is") # and its object carries the message provided. - now = cloned_repo.create_tag("now") # This is a tag-reference. It may not carry meta-data + now = cloned_repo.create_tag("now") # This is a tag-reference. It may not carry meta-data. assert now.tag is None # ![10-test_init_repo_object] # Object handling # [11-test_init_repo_object] assert now.commit.message != past.commit.message - # You can read objects directly through binary streams, no working tree required + # You can read objects directly through binary streams, no working tree required. assert (now.commit.tree / "VERSION").data_stream.read().decode("ascii").startswith("3") - # You can traverse trees as well to handle all contained files of a particular commit + # You can traverse trees as well to handle all contained files of a particular commit. file_count = 0 tree_count = 0 tree = past.commit.tree for item in tree.traverse(): file_count += item.type == "blob" tree_count += item.type == "tree" - assert file_count and tree_count # we have accumulated all directories and files - self.assertEqual(len(tree.blobs) + len(tree.trees), len(tree)) # a tree is iterable on its children + assert file_count and tree_count # We have accumulated all directories and files. + self.assertEqual(len(tree.blobs) + len(tree.trees), len(tree)) # A tree is iterable on its children. # ![11-test_init_repo_object] - # remotes allow handling push, pull and fetch operations + # Remotes allow handling push, pull and fetch operations. # [12-test_init_repo_object] from git import RemoteProgress @@ -147,67 +148,67 @@ def update(self, op_code, cur_count, max_count=None, message=""): # end - self.assertEqual(len(cloned_repo.remotes), 1) # we have been cloned, so should be one remote - self.assertEqual(len(bare_repo.remotes), 0) # this one was just initialized + self.assertEqual(len(cloned_repo.remotes), 1) # We have been cloned, so should be one remote. + self.assertEqual(len(bare_repo.remotes), 0) # This one was just initialized. origin = bare_repo.create_remote("origin", url=cloned_repo.working_tree_dir) assert origin.exists() for fetch_info in origin.fetch(progress=MyProgressPrinter()): print("Updated %s to %s" % (fetch_info.ref, fetch_info.commit)) - # create a local branch at the latest fetched master. We specify the name statically, but you have all + # Create a local branch at the latest fetched master. We specify the name statically, but you have all # information to do it programmatically as well. bare_master = bare_repo.create_head("master", origin.refs.master) bare_repo.head.set_reference(bare_master) assert not bare_repo.delete_remote(origin).exists() - # push and pull behave very similarly + # push and pull behave very similarly. # ![12-test_init_repo_object] # index # [13-test_init_repo_object] - self.assertEqual(new_branch.checkout(), cloned_repo.active_branch) # checking out branch adjusts the wtree - self.assertEqual(new_branch.commit, past.commit) # Now the past is checked out + self.assertEqual(new_branch.checkout(), cloned_repo.active_branch) # Checking out branch adjusts the wtree. + self.assertEqual(new_branch.commit, past.commit) # Now the past is checked out. new_file_path = os.path.join(cloned_repo.working_tree_dir, "my-new-file") - open(new_file_path, "wb").close() # create new file in working tree - cloned_repo.index.add([new_file_path]) # add it to the index - # Commit the changes to deviate masters history + open(new_file_path, "wb").close() # Create new file in working tree. + cloned_repo.index.add([new_file_path]) # Add it to the index. + # Commit the changes to deviate masters history. cloned_repo.index.commit("Added a new file in the past - for later merge") - # prepare a merge - master = cloned_repo.heads.master # right-hand side is ahead of us, in the future - merge_base = cloned_repo.merge_base(new_branch, master) # allows for a three-way merge - cloned_repo.index.merge_tree(master, base=merge_base) # write the merge result into index + # Prepare a merge. + master = cloned_repo.heads.master # Right-hand side is ahead of us, in the future. + merge_base = cloned_repo.merge_base(new_branch, master) # Allows for a three-way merge. + cloned_repo.index.merge_tree(master, base=merge_base) # Write the merge result into index. cloned_repo.index.commit( "Merged past and now into future ;)", parent_commits=(new_branch.commit, master.commit), ) - # now new_branch is ahead of master, which probably should be checked out and reset softly. - # note that all these operations didn't touch the working tree, as we managed it ourselves. - # This definitely requires you to know what you are doing :) ! - assert os.path.basename(new_file_path) in new_branch.commit.tree # new file is now in tree - master.commit = new_branch.commit # let master point to most recent commit - cloned_repo.head.reference = master # we adjusted just the reference, not the working tree or index + # Now new_branch is ahead of master, which probably should be checked out and reset softly. + # Note that all these operations didn't touch the working tree, as we managed it ourselves. + # This definitely requires you to know what you are doing! :) + assert os.path.basename(new_file_path) in new_branch.commit.tree # New file is now in tree. + master.commit = new_branch.commit # Let master point to most recent commit. + cloned_repo.head.reference = master # We adjusted just the reference, not the working tree or index. # ![13-test_init_repo_object] # submodules # [14-test_init_repo_object] - # create a new submodule and check it out on the spot, setup to track master branch of `bare_repo` + # Create a new submodule and check it out on the spot, setup to track master branch of `bare_repo`. # As our GitPython repository has submodules already that point to GitHub, make sure we don't - # interact with them + # interact with them. for sm in cloned_repo.submodules: assert not sm.remove().exists() # after removal, the sm doesn't exist anymore sm = cloned_repo.create_submodule("mysubrepo", "path/to/subrepo", url=bare_repo.git_dir, branch="master") - # .gitmodules was written and added to the index, which is now being committed + # .gitmodules was written and added to the index, which is now being committed. cloned_repo.index.commit("Added submodule") - assert sm.exists() and sm.module_exists() # this submodule is definitely available - sm.remove(module=True, configuration=False) # remove the working tree - assert sm.exists() and not sm.module_exists() # the submodule itself is still available + assert sm.exists() and sm.module_exists() # This submodule is definitely available. + sm.remove(module=True, configuration=False) # Remove the working tree. + assert sm.exists() and not sm.module_exists() # The submodule itself is still available. - # update all submodules, non-recursively to save time, this method is very powerful, go have a look + # Update all submodules, non-recursively to save time. This method is very powerful, go have a look. cloned_repo.submodule_update(recursive=False) - assert sm.module_exists() # The submodules working tree was checked out by update + assert sm.module_exists() # The submodule's working tree was checked out by update. # ![14-test_init_repo_object] @with_rw_directory @@ -218,25 +219,25 @@ def test_references_and_objects(self, rw_dir): repo = git.Repo.clone_from(self._small_repo_url(), os.path.join(rw_dir, "repo"), branch="master") heads = repo.heads - master = heads.master # lists can be accessed by name for convenience - master.commit # the commit pointed to by head called master - master.rename("new_name") # rename heads + master = heads.master # Lists can be accessed by name for convenience. + master.commit # the commit pointed to by head called master. + master.rename("new_name") # Rename heads. master.rename("master") # ![1-test_references_and_objects] # [2-test_references_and_objects] tags = repo.tags tagref = tags[0] - tagref.tag # tags may have tag objects carrying additional information - tagref.commit # but they always point to commits - repo.delete_tag(tagref) # delete or - repo.create_tag("my_tag") # create tags using the repo for convenience + tagref.tag # Tags may have tag objects carrying additional information + tagref.commit # but they always point to commits. + repo.delete_tag(tagref) # Delete or + repo.create_tag("my_tag") # create tags using the repo for convenience. # ![2-test_references_and_objects] # [3-test_references_and_objects] - head = repo.head # the head points to the active branch/ref - master = head.reference # retrieve the reference the head points to - master.commit # from here you use it as any other reference + head = repo.head # The head points to the active branch/ref. + master = head.reference # Retrieve the reference the head points to. + master.commit # From here you use it as any other reference. # ![3-test_references_and_objects] # # [4-test_references_and_objects] @@ -246,14 +247,14 @@ def test_references_and_objects(self, rw_dir): # ![4-test_references_and_objects] # [5-test_references_and_objects] - new_branch = repo.create_head("new") # create a new one - new_branch.commit = "HEAD~10" # set branch to another commit without changing index or working trees - repo.delete_head(new_branch) # delete an existing head - only works if it is not checked out + new_branch = repo.create_head("new") # Create a new one. + new_branch.commit = "HEAD~10" # Set branch to another commit without changing index or working trees. + repo.delete_head(new_branch) # Delete an existing head - only works if it is not checked out. # ![5-test_references_and_objects] # [6-test_references_and_objects] new_tag = repo.create_tag("my_new_tag", message="my message") - # You cannot change the commit a tag points to. Tags need to be re-created + # You cannot change the commit a tag points to. Tags need to be re-created. self.assertRaises(AttributeError, setattr, new_tag, "commit", repo.commit("HEAD~1")) repo.delete_tag(new_tag) # ![6-test_references_and_objects] @@ -272,22 +273,22 @@ def test_references_and_objects(self, rw_dir): # ![8-test_references_and_objects] # [9-test_references_and_objects] - self.assertEqual(hct.type, "tree") # preset string type, being a class attribute + self.assertEqual(hct.type, "tree") # Preset string type, being a class attribute. assert hct.size > 0 # size in bytes assert len(hct.hexsha) == 40 assert len(hct.binsha) == 20 # ![9-test_references_and_objects] # [10-test_references_and_objects] - self.assertEqual(hct.path, "") # root tree has no path - assert hct.trees[0].path != "" # the first contained item has one though - self.assertEqual(hct.mode, 0o40000) # trees have the mode of a linux directory - self.assertEqual(hct.blobs[0].mode, 0o100644) # blobs have specific mode, comparable to a standard linux fs + self.assertEqual(hct.path, "") # Root tree has no path. + assert hct.trees[0].path != "" # The first contained item has one though. + self.assertEqual(hct.mode, 0o40000) # Trees have the mode of a Linux directory. + self.assertEqual(hct.blobs[0].mode, 0o100644) # Blobs have specific mode, comparable to a standard Linux fs. # ![10-test_references_and_objects] # [11-test_references_and_objects] - hct.blobs[0].data_stream.read() # stream object to read data from - hct.blobs[0].stream_data(open(os.path.join(rw_dir, "blob_data"), "wb")) # write data to given stream + hct.blobs[0].data_stream.read() # Stream object to read data from. + hct.blobs[0].stream_data(open(os.path.join(rw_dir, "blob_data"), "wb")) # Write data to a given stream. # ![11-test_references_and_objects] # [12-test_references_and_objects] @@ -299,7 +300,7 @@ def test_references_and_objects(self, rw_dir): # [13-test_references_and_objects] fifty_first_commits = list(repo.iter_commits("master", max_count=50)) assert len(fifty_first_commits) == 50 - # this will return commits 21-30 from the commit list as traversed backwards master + # This will return commits 21-30 from the commit list as traversed backwards master. ten_commits_past_twenty = list(repo.iter_commits("master", max_count=10, skip=20)) assert len(ten_commits_past_twenty) == 10 assert fifty_first_commits[20:30] == ten_commits_past_twenty @@ -334,20 +335,20 @@ def test_references_and_objects(self, rw_dir): # ![17-test_references_and_objects] # [18-test_references_and_objects] - assert len(tree.trees) > 0 # trees are subdirectories - assert len(tree.blobs) > 0 # blobs are files + assert len(tree.trees) > 0 # Trees are subdirectories. + assert len(tree.blobs) > 0 # Blobs are files. assert len(tree.blobs) + len(tree.trees) == len(tree) # ![18-test_references_and_objects] # [19-test_references_and_objects] - self.assertEqual(tree["smmap"], tree / "smmap") # access by index and by sub-path - for entry in tree: # intuitive iteration of tree members + self.assertEqual(tree["smmap"], tree / "smmap") # Access by index and by sub-path. + for entry in tree: # Intuitive iteration of tree members. print(entry) - blob = tree.trees[1].blobs[0] # let's get a blob in a sub-tree + blob = tree.trees[1].blobs[0] # Let's get a blob in a sub-tree. assert blob.name assert len(blob.path) < len(blob.abspath) - self.assertEqual(tree.trees[1].name + "/" + blob.name, blob.path) # this is how relative blob path generated - self.assertEqual(tree[blob.path], blob) # you can use paths like 'dir/file' in tree + self.assertEqual(tree.trees[1].name + "/" + blob.name, blob.path) # This is how relative blob path generated. + self.assertEqual(tree[blob.path], blob) # You can use paths like 'dir/file' in tree, # ![19-test_references_and_objects] # [20-test_references_and_objects] @@ -356,11 +357,11 @@ def test_references_and_objects(self, rw_dir): # ![20-test_references_and_objects] # [21-test_references_and_objects] - # This example shows the various types of allowed ref-specs + # This example shows the various types of allowed ref-specs. assert repo.tree() == repo.head.commit.tree past = repo.commit("HEAD~5") assert repo.tree(past) == repo.tree(past.hexsha) - self.assertEqual(repo.tree("v0.8.1").type, "tree") # yes, you can provide any refspec - works everywhere + self.assertEqual(repo.tree("v0.8.1").type, "tree") # Yes, you can provide any refspec - works everywhere. # ![21-test_references_and_objects] # [22-test_references_and_objects] @@ -369,36 +370,36 @@ def test_references_and_objects(self, rw_dir): # [23-test_references_and_objects] index = repo.index - # The index contains all blobs in a flat list + # The index contains all blobs in a flat list. assert len(list(index.iter_blobs())) == len([o for o in repo.head.commit.tree.traverse() if o.type == "blob"]) - # Access blob objects + # Access blob objects. for (_path, _stage), _entry in index.entries.items(): pass new_file_path = os.path.join(repo.working_tree_dir, "new-file-name") open(new_file_path, "w").close() - index.add([new_file_path]) # add a new file to the index - index.remove(["LICENSE"]) # remove an existing one - assert os.path.isfile(os.path.join(repo.working_tree_dir, "LICENSE")) # working tree is untouched + index.add([new_file_path]) # Add a new file to the index. + index.remove(["LICENSE"]) # Remove an existing one. + assert os.path.isfile(os.path.join(repo.working_tree_dir, "LICENSE")) # Working tree is untouched. - self.assertEqual(index.commit("my commit message").type, "commit") # commit changed index - repo.active_branch.commit = repo.commit("HEAD~1") # forget last commit + self.assertEqual(index.commit("my commit message").type, "commit") # Commit changed index. + repo.active_branch.commit = repo.commit("HEAD~1") # Forget last commit. from git import Actor author = Actor("An author", "author@example.com") committer = Actor("A committer", "committer@example.com") - # commit by commit message and author and committer + # Commit with a commit message, author, and committer. index.commit("my commit message", author=author, committer=committer) # ![23-test_references_and_objects] # [24-test_references_and_objects] from git import IndexFile - # loads a tree into a temporary index, which exists just in memory + # Load a tree into a temporary index, which exists just in memory. IndexFile.from_tree(repo, "HEAD~1") - # merge two trees three-way into memory + # Merge two trees three-way into memory... merge_index = IndexFile.from_tree(repo, "HEAD~10", "HEAD", repo.merge_base("HEAD~10", "HEAD")) - # and persist it + # ...and persist it. merge_index.write(os.path.join(rw_dir, "merged_index")) # ![24-test_references_and_objects] @@ -407,20 +408,20 @@ def test_references_and_objects(self, rw_dir): origin = empty_repo.create_remote("origin", repo.remotes.origin.url) assert origin.exists() assert origin == empty_repo.remotes.origin == empty_repo.remotes["origin"] - origin.fetch() # assure we actually have data. fetch() returns useful information - # Setup a local tracking branch of a remote branch - empty_repo.create_head("master", origin.refs.master) # create local branch "master" from remote "master" - empty_repo.heads.master.set_tracking_branch(origin.refs.master) # set local "master" to track remote "master - empty_repo.heads.master.checkout() # checkout local "master" to working tree + origin.fetch() # Ensure we actually have data. fetch() returns useful information. + # Set up a local tracking branch of a remote branch. + empty_repo.create_head("master", origin.refs.master) # Create local branch "master" from remote "master". + empty_repo.heads.master.set_tracking_branch(origin.refs.master) # Set local "master" to track remote "master. + empty_repo.heads.master.checkout() # Check out local "master" to working tree. # Three above commands in one: empty_repo.create_head("master", origin.refs.master).set_tracking_branch(origin.refs.master).checkout() - # rename remotes + # Rename remotes. origin.rename("new_origin") - # push and pull behaves similarly to `git push|pull` + # Push and pull behaves similarly to `git push|pull`. origin.pull() - origin.push() # attempt push, ignore errors - origin.push().raise_if_error() # push and raise error if it fails - # assert not empty_repo.delete_remote(origin).exists() # create and delete remotes + origin.push() # Attempt push, ignore errors. + origin.push().raise_if_error() # Push and raise error if it fails. + # assert not empty_repo.delete_remote(origin).exists() # Create and delete remotes. # ![25-test_references_and_objects] # [26-test_references_and_objects] @@ -428,20 +429,20 @@ def test_references_and_objects(self, rw_dir): with origin.config_writer as cw: cw.set("pushurl", "other_url") - # Please note that in python 2, writing origin.config_writer.set(...) is totally safe. + # Please note that in Python 2, writing origin.config_writer.set(...) is totally safe. # In py3 __del__ calls can be delayed, thus not writing changes in time. # ![26-test_references_and_objects] # [27-test_references_and_objects] hcommit = repo.head.commit - hcommit.diff() # diff tree against index - hcommit.diff("HEAD~1") # diff tree against previous tree - hcommit.diff(None) # diff tree against working tree + hcommit.diff() # diff tree against index. + hcommit.diff("HEAD~1") # diff tree against previous tree. + hcommit.diff(None) # diff tree against working tree. index = repo.index - index.diff() # diff index against itself yielding empty diff - index.diff(None) # diff index against working copy - index.diff("HEAD") # diff index against current HEAD tree + index.diff() # diff index against itself yielding empty diff. + index.diff(None) # diff index against working copy. + index.diff("HEAD") # diff index against current HEAD tree. # ![27-test_references_and_objects] # [28-test_references_and_objects] @@ -451,32 +452,32 @@ def test_references_and_objects(self, rw_dir): # ![28-test_references_and_objects] # [29-test_references_and_objects] - # Reset our working tree 10 commits into the past + # Reset our working tree 10 commits into the past. past_branch = repo.create_head("past_branch", "HEAD~10") repo.head.reference = past_branch assert not repo.head.is_detached - # reset the index and working tree to match the pointed-to commit + # Reset the index and working tree to match the pointed-to commit. repo.head.reset(index=True, working_tree=True) - # To detach your head, you have to point to a commit directly + # To detach your head, you have to point to a commit directly. repo.head.reference = repo.commit("HEAD~5") assert repo.head.is_detached - # now our head points 15 commits into the past, whereas the working tree - # and index are 10 commits in the past + # Now our head points 15 commits into the past, whereas the working tree + # and index are 10 commits in the past. # ![29-test_references_and_objects] # [30-test_references_and_objects] - # checkout the branch using git-checkout. It will fail as the working tree appears dirty + # Check out the branch using git-checkout. It will fail as the working tree appears dirty. self.assertRaises(git.GitCommandError, repo.heads.master.checkout) repo.heads.past_branch.checkout() # ![30-test_references_and_objects] # [31-test_references_and_objects] git = repo.git - git.checkout("HEAD", b="my_new_branch") # create a new branch + git.checkout("HEAD", b="my_new_branch") # Create a new branch. git.branch("another-new-one") - git.branch("-D", "another-new-one") # pass strings for full control over argument order - git.for_each_ref() # '-' becomes '_' when calling it + git.branch("-D", "another-new-one") # Pass strings for full control over argument order. + git.for_each_ref() # '-' becomes '_' when calling it. # ![31-test_references_and_objects] repo.git.clear_cache() @@ -493,19 +494,19 @@ def test_submodules(self): assert len(sms) == 1 sm = sms[0] - self.assertEqual(sm.name, "gitdb") # git-python has gitdb as single submodule ... - self.assertEqual(sm.children()[0].name, "smmap") # ... which has smmap as single submodule + self.assertEqual(sm.name, "gitdb") # GitPython has gitdb as its one and only (direct) submodule... + self.assertEqual(sm.children()[0].name, "smmap") # ...which has smmap as its one and only submodule. - # The module is the repository referenced by the submodule - assert sm.module_exists() # the module is available, which doesn't have to be the case. + # The module is the repository referenced by the submodule. + assert sm.module_exists() # The module is available, which doesn't have to be the case. assert sm.module().working_tree_dir.endswith("gitdb") - # the submodule's absolute path is the module's path + # The submodule's absolute path is the module's path. assert sm.abspath == sm.module().working_tree_dir - self.assertEqual(len(sm.hexsha), 40) # Its sha defines the commit to checkout - assert sm.exists() # yes, this submodule is valid and exists - # read its configuration conveniently + self.assertEqual(len(sm.hexsha), 40) # Its sha defines the commit to check out. + assert sm.exists() # Yes, this submodule is valid and exists. + # Read its configuration conveniently. assert sm.config_reader().get_value("path") == sm.path - self.assertEqual(len(sm.children()), 1) # query the submodule hierarchy + self.assertEqual(len(sm.children()), 1) # Query the submodule hierarchy. # ![1-test_submodules] @with_rw_directory @@ -516,7 +517,7 @@ def test_add_file_and_commit(self, rw_dir): file_name = os.path.join(repo_dir, "new-file") r = git.Repo.init(repo_dir) - # This function just creates an empty file ... + # This function just creates an empty file. open(file_name, "wb").close() r.index.add([file_name]) r.index.commit("initial commit") diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 342a7f293..13b587bd5 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -28,7 +28,7 @@ def test_init_repo_object(self, path_to_dir): def test_cloned_repo_object(self, local_dir): from git import Repo - # code to clone from url + # Code to clone from url # [1-test_cloned_repo_object] # $ git clone @@ -37,7 +37,7 @@ def test_cloned_repo_object(self, local_dir): repo = Repo.clone_from(repo_url, local_dir) # ![1-test_cloned_repo_object] - # code to add files + # Code to add files # [2-test_cloned_repo_object] # We must make a change to a file so that we can add the update to git @@ -52,7 +52,7 @@ def test_cloned_repo_object(self, local_dir): repo.index.add(add_file) # notice the add function requires a list of paths # ![3-test_cloned_repo_object] - # code to commit - not sure how to test this + # Code to commit - not sure how to test this # [4-test_cloned_repo_object] # $ git commit -m repo.index.commit("Update to file2") @@ -61,8 +61,8 @@ def test_cloned_repo_object(self, local_dir): # [5-test_cloned_repo_object] # $ git log - # relative path from git root - repo.iter_commits(all=True, max_count=10, paths=update_file) # gets the last 10 commits from all branches + # Relative path from git root + repo.iter_commits(all=True, max_count=10, paths=update_file) # Gets the last 10 commits from all branches. # Outputs: @@ -79,7 +79,7 @@ def test_cloned_repo_object(self, local_dir): # Untracked files - create new file # [7-test_cloned_repo_object] - f = open(f"{local_dir}/untracked.txt", "w") # creates an empty file + f = open(f"{local_dir}/untracked.txt", "w") # Creates an empty file. f.close() # ![7-test_cloned_repo_object] @@ -90,14 +90,14 @@ def test_cloned_repo_object(self, local_dir): # Modified files # [9-test_cloned_repo_object] - # Let's modify one of our tracked files + # Let's modify one of our tracked files. with open(f"{local_dir}/Downloads/file3.txt", "w") as f: - f.write("file3 version 2") # overwrite file 3 + f.write("file3 version 2") # Overwrite file 3. # ![9-test_cloned_repo_object] # [10-test_cloned_repo_object] - repo.index.diff(None) # compares staging area to working directory + repo.index.diff(None) # Compares staging area to working directory. # Output: [, # ] @@ -112,7 +112,7 @@ def test_cloned_repo_object(self, local_dir): # Downloads/file3.txt # ![11-test_cloned_repo_object] - # compares staging area to head commit + # Compares staging area to head commit # [11.1-test_cloned_repo_object] diffs = repo.index.diff(repo.head.commit) for d in diffs: @@ -122,7 +122,7 @@ def test_cloned_repo_object(self, local_dir): # ![11.1-test_cloned_repo_object] # [11.2-test_cloned_repo_object] - # lets add untracked.txt + # Let's add untracked.txt. repo.index.add(["untracked.txt"]) diffs = repo.index.diff(repo.head.commit) for d in diffs: @@ -152,7 +152,7 @@ def test_cloned_repo_object(self, local_dir): # Previous commit tree # [13-test_cloned_repo_object] - prev_commits = list(repo.iter_commits(all=True, max_count=10)) # last 10 commits from all branches + prev_commits = list(repo.iter_commits(all=True, max_count=10)) # Last 10 commits from all branches. tree = prev_commits[0].tree # ![13-test_cloned_repo_object] @@ -191,29 +191,29 @@ def print_files_from_git(root, level=0): # Printing text files # [17-test_cloned_repo_object] print_file = "dir1/file2.txt" - tree[print_file] # the head commit tree + tree[print_file] # The head commit tree. # Output # ![17-test_cloned_repo_object] - # print latest file + # Print latest file # [18-test_cloned_repo_object] blob = tree[print_file] print(blob.data_stream.read().decode()) # Output - # file 2 version 1 + # File 2 version 1 # Update version 2 # ![18-test_cloned_repo_object] - # print previous tree + # Print previous tree # [18.1-test_cloned_repo_object] commits_for_file = list(repo.iter_commits(all=True, paths=print_file)) - tree = commits_for_file[-1].tree # gets the first commit tree + tree = commits_for_file[-1].tree # Gets the first commit tree. blob = tree[print_file] print(blob.data_stream.read().decode()) # Output - # file 2 version 1 + # File 2 version 1 # ![18.1-test_cloned_repo_object] From 11fce8a3a7ccd82fbfde5aa51dd62dcb9e76c2f3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 06:04:55 -0400 Subject: [PATCH 0813/1790] Revise docstrings/comments in most other test modules Intentionally untouched test modules in this commit: - Modules where there are no such changes to be made. - Modules in test/performance/ -- will be done separately. - test/lib/ modules (these are test *helpers* and will be done separately). - test/test_util.py, because although it has a few comments that might be improved, it may make more sense to do that together with some further refactoring changes. --- test/test_actor.py | 2 +- test/test_base.py | 21 +-- test/test_blob_filter.py | 1 + test/test_clone.py | 2 +- test/test_commit.py | 86 ++++++----- test/test_config.py | 44 +++--- test/test_db.py | 5 +- test/test_diff.py | 57 +++---- test/test_fun.py | 68 ++++----- test/test_git.py | 35 ++--- test/test_index.py | 210 +++++++++++++------------ test/test_reflog.py | 30 ++-- test/test_refs.py | 158 ++++++++++--------- test/test_remote.py | 191 +++++++++++------------ test/test_repo.py | 178 ++++++++++----------- test/test_submodule.py | 323 ++++++++++++++++++++------------------- test/test_tree.py | 20 +-- 17 files changed, 719 insertions(+), 712 deletions(-) diff --git a/test/test_actor.py b/test/test_actor.py index f495ac084..80b93d7bc 100644 --- a/test/test_actor.py +++ b/test/test_actor.py @@ -14,7 +14,7 @@ def test_from_string_should_separate_name_and_email(self): self.assertEqual("Michael Trier", a.name) self.assertEqual("mtrier@example.com", a.email) - # base type capabilities + # Base type capabilities assert a == a assert not (a != a) m = set() diff --git a/test/test_base.py b/test/test_base.py index 90e701c4b..94a268ecd 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -4,6 +4,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import os import sys import tempfile @@ -34,7 +35,7 @@ def tearDown(self): ) def test_base_object(self): - # test interface of base object classes + # Test interface of base object classes. types = (Blob, Tree, Commit, TagObject) self.assertEqual(len(types), len(self.type_tuples)) @@ -61,12 +62,12 @@ def test_base_object(self): if isinstance(item, base.IndexObject): num_index_objs += 1 - if hasattr(item, "path"): # never runs here - assert not item.path.startswith("/") # must be relative + if hasattr(item, "path"): # Never runs here. + assert not item.path.startswith("/") # Must be relative. assert isinstance(item.mode, int) # END index object check - # read from stream + # Read from stream. data_stream = item.data_stream data = data_stream.read() assert data @@ -79,7 +80,7 @@ def test_base_object(self): os.remove(tmpfilename) # END for each object type to create - # each has a unique sha + # Each has a unique sha. self.assertEqual(len(s), num_objs) self.assertEqual(len(s | s), num_objs) self.assertEqual(num_index_objs, 2) @@ -92,7 +93,7 @@ def test_get_object_type_by_name(self): self.assertRaises(ValueError, get_object_type_by_name, b"doesntexist") def test_object_resolution(self): - # objects must be resolved to shas so they compare equal + # Objects must be resolved to shas so they compare equal. self.assertEqual(self.rorepo.head.reference.object, self.rorepo.active_branch.object) @with_rw_repo("HEAD", bare=True) @@ -122,7 +123,7 @@ def test_add_unicode(self, rw_repo): file_path = osp.join(rw_repo.working_dir, filename) - # verify first that we could encode file name in this environment + # Verify first that we could encode file name in this environment. try: file_path.encode(sys.getfilesystemencoding()) except UnicodeEncodeError as e: @@ -132,14 +133,14 @@ def test_add_unicode(self, rw_repo): fp.write(b"something") if is_win: - # on windows, there is no way this works, see images on + # On Windows, there is no way this works, see images on: # https://github.com/gitpython-developers/GitPython/issues/147#issuecomment-68881897 - # Therefore, it must be added using the python implementation + # Therefore, it must be added using the Python implementation. rw_repo.index.add([file_path]) # However, when the test winds down, rmtree fails to delete this file, which is recognized # as ??? only. else: - # on posix, we can just add unicode files without problems + # On POSIX, we can just add Unicode files without problems. rw_repo.git.add(rw_repo.working_dir) # end rw_repo.index.commit("message") diff --git a/test/test_blob_filter.py b/test/test_blob_filter.py index cbaa30b8b..ad4f0e7ff 100644 --- a/test/test_blob_filter.py +++ b/test/test_blob_filter.py @@ -1,4 +1,5 @@ """Test the blob filter.""" + from pathlib import Path from typing import Sequence, Tuple from unittest.mock import MagicMock diff --git a/test/test_clone.py b/test/test_clone.py index 1b4a6c332..f66130cf0 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -21,7 +21,7 @@ def test_checkout_in_non_empty_dir(self, rw_dir): garbage_file.write_text("Garbage!") # Verify that cloning into the non-empty dir fails while complaining about - # the target directory not being empty/non-existent + # the target directory not being empty/non-existent. try: self.rorepo.clone(non_empty_dir) except git.GitCommandError as exc: diff --git a/test/test_commit.py b/test/test_commit.py index 527aea334..3f3c52f1b 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -4,6 +4,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import copy from datetime import datetime from io import BytesIO @@ -28,18 +29,20 @@ class TestCommitSerialization(TestBase): def assert_commit_serialization(self, rwrepo, commit_id, print_performance_info=False): - """traverse all commits in the history of commit identified by commit_id and check + """Traverse all commits in the history of commit identified by commit_id and check if the serialization works. - :param print_performance_info: if True, we will show how fast we are""" - ns = 0 # num serializations - nds = 0 # num deserializations + + :param print_performance_info: If True, we will show how fast we are. + """ + ns = 0 # Number of serializations. + nds = 0 # Number of deserializations. st = time.time() for cm in rwrepo.commit(commit_id).traverse(): nds += 1 - # assert that we deserialize commits correctly, hence we get the same - # sha on serialization + # Assert that we deserialize commits correctly, hence we get the same + # sha on serialization. stream = BytesIO() cm._serialize(stream) ns += 1 @@ -71,13 +74,13 @@ def assert_commit_serialization(self, rwrepo, commit_id, print_performance_info= streamlen = stream.tell() stream.seek(0) - # reuse istream + # Reuse istream. istream.size = streamlen istream.stream = stream istream.binsha = None nc.binsha = rwrepo.odb.store(istream).binsha - # if it worked, we have exactly the same contents ! + # If it worked, we have exactly the same contents! self.assertEqual(nc.hexsha, cm.hexsha) # END check commits elapsed = time.time() - st @@ -94,7 +97,7 @@ def assert_commit_serialization(self, rwrepo, commit_id, print_performance_info= class TestCommit(TestCommitSerialization): def test_bake(self): commit = self.rorepo.commit("2454ae89983a4496a445ce347d7a41c0bb0ea7ae") - # commits have no dict + # Commits have no dict. self.assertRaises(AttributeError, setattr, commit, "someattr", 1) commit.author # bake @@ -148,7 +151,7 @@ def check_entries(d): check_entries(d) # END for each stated file - # assure data is parsed properly + # Check that data is parsed properly. michael = Actor._from_string("Michael Trier ") self.assertEqual(commit.author, michael) self.assertEqual(commit.committer, michael) @@ -162,9 +165,9 @@ def test_renames(self): commit = self.rorepo.commit("185d847ec7647fd2642a82d9205fb3d07ea71715") files = commit.stats.files - # when a file is renamed, the output of git diff is like "dir/{old => new}" - # unless we disable rename with --no-renames, which produces two lines - # one with the old path deletes and another with the new added + # When a file is renamed, the output of git diff is like "dir/{old => new}" + # unless we disable rename with --no-renames, which produces two lines, + # one with the old path deletes and another with the new added. self.assertEqual(len(files), 2) def check_entries(path, changes): @@ -190,7 +193,7 @@ def check_entries(path, changes): # END for each stated file def test_unicode_actor(self): - # assure we can parse unicode actors correctly + # Check that we can parse Unicode actors correctly. name = "Üäöß ÄußÉ" self.assertEqual(len(name), 9) special = Actor._from_string("%s " % name) @@ -205,7 +208,7 @@ def test_traversal(self): p00 = p0.parents[0] p10 = p1.parents[0] - # basic branch first, depth first + # Basic branch first, depth first. dfirst = start.traverse(branch_first=False) bfirst = start.traverse(branch_first=True) self.assertEqual(next(dfirst), p0) @@ -216,7 +219,7 @@ def test_traversal(self): self.assertEqual(next(bfirst), p00) self.assertEqual(next(bfirst), p10) - # at some point, both iterations should stop + # At some point, both iterations should stop. self.assertEqual(list(bfirst)[-1], first) stoptraverse = self.rorepo.commit("254d04aa3180eb8b8daf7b7ff25f010cd69b4e7d").traverse( @@ -235,40 +238,39 @@ def test_traversal(self): stoptraverse = self.rorepo.commit("254d04aa3180eb8b8daf7b7ff25f010cd69b4e7d").traverse(as_edge=True) self.assertEqual(len(next(stoptraverse)), 2) - # ignore self + # Ignore self self.assertEqual(next(start.traverse(ignore_self=False)), start) - # depth + # Depth self.assertEqual(len(list(start.traverse(ignore_self=False, depth=0))), 1) - # prune + # Prune self.assertEqual(next(start.traverse(branch_first=1, prune=lambda i, d: i == p0)), p1) - # predicate + # Predicate self.assertEqual(next(start.traverse(branch_first=1, predicate=lambda i, d: i == p1)), p1) - # traversal should stop when the beginning is reached + # Traversal should stop when the beginning is reached. self.assertRaises(StopIteration, next, first.traverse()) - # parents of the first commit should be empty ( as the only parent has a null - # sha ) + # Parents of the first commit should be empty (as the only parent has a null sha) self.assertEqual(len(first.parents), 0) def test_iteration(self): - # we can iterate commits + # We can iterate commits. all_commits = Commit.list_items(self.rorepo, self.rorepo.head) assert all_commits self.assertEqual(all_commits, list(self.rorepo.iter_commits())) - # this includes merge commits + # This includes merge commits. mcomit = self.rorepo.commit("d884adc80c80300b4cc05321494713904ef1df2d") assert mcomit in all_commits - # we can limit the result to paths + # We can limit the result to paths. ltd_commits = list(self.rorepo.iter_commits(paths="CHANGES")) assert ltd_commits and len(ltd_commits) < len(all_commits) - # show commits of multiple paths, resulting in a union of commits + # Show commits of multiple paths, resulting in a union of commits. less_ltd_commits = list(Commit.iter_items(self.rorepo, "master", paths=("CHANGES", "AUTHORS"))) assert len(ltd_commits) < len(less_ltd_commits) @@ -280,7 +282,7 @@ def __init__(self, *args, **kwargs): assert type(child_commits[0]) is Child def test_iter_items(self): - # pretty not allowed + # pretty not allowed. self.assertRaises(ValueError, Commit.iter_items, self.rorepo, "master", pretty="raw") def test_rev_list_bisect_all(self): @@ -311,14 +313,14 @@ def test_ambiguous_arg_iteration(self, rw_dir): touch(path) rw_repo.index.add([path]) rw_repo.index.commit("initial commit") - list(rw_repo.iter_commits(rw_repo.head.ref)) # should fail unless bug is fixed + list(rw_repo.iter_commits(rw_repo.head.ref)) # Should fail unless bug is fixed. def test_count(self): self.assertEqual(self.rorepo.tag("refs/tags/0.1.5").commit.count(), 143) def test_list(self): # This doesn't work anymore, as we will either attempt getattr with bytes, or compare 20 byte string - # with actual 20 byte bytes. This usage makes no sense anyway + # with actual 20 byte bytes. This usage makes no sense anyway. assert isinstance( Commit.list_items(self.rorepo, "0.1.5", max_count=5)["5117c9c8a4d3af19a9958677e45cda9269de1541"], Commit, @@ -340,7 +342,7 @@ def test_equality(self): self.assertNotEqual(commit2, commit3) def test_iter_parents(self): - # should return all but ourselves, even if skip is defined + # Should return all but ourselves, even if skip is defined. c = self.rorepo.commit("0.1.5") for skip in (0, 1): piter = c.iter_parents(skip=skip) @@ -355,17 +357,17 @@ def test_name_rev(self): @with_rw_repo("HEAD", bare=True) def test_serialization(self, rwrepo): - # create all commits of our repo + # Create all commits of our repo. self.assert_commit_serialization(rwrepo, "0.1.6") def test_serialization_unicode_support(self): self.assertEqual(Commit.default_encoding.lower(), "utf-8") - # create a commit with unicode in the message, and the author's name - # Verify its serialization and deserialization + # Create a commit with Unicode in the message, and the author's name. + # Verify its serialization and deserialization. cmt = self.rorepo.commit("0.1.6") - assert isinstance(cmt.message, str) # it automatically decodes it as such - assert isinstance(cmt.author.name, str) # same here + assert isinstance(cmt.message, str) # It automatically decodes it as such. + assert isinstance(cmt.author.name, str) # Same here. cmt.message = "üäêèß" self.assertEqual(len(cmt.message), 5) @@ -383,8 +385,8 @@ def test_serialization_unicode_support(self): self.assertEqual(cmt.author.name, ncmt.author.name) self.assertEqual(cmt.message, ncmt.message) - # actually, it can't be printed in a shell as repr wants to have ascii only - # it appears + # Actually, it can't be printed in a shell as repr wants to have ascii only + # it appears. cmt.author.__repr__() def test_invalid_commit(self): @@ -498,14 +500,14 @@ def test_trailers(self): KEY_2 = "Key" VALUE_2 = "Value with inner spaces" - # Check the following trailer example is extracted from multiple msg variations + # Check that the following trailer example is extracted from multiple msg variations. TRAILER = f"{KEY_1}: {VALUE_1_1}\n{KEY_2}: {VALUE_2}\n{KEY_1}: {VALUE_1_2}" msgs = [ f"Subject\n\n{TRAILER}\n", f"Subject\n \nSome body of a function\n \n{TRAILER}\n", f"Subject\n \nSome body of a function\n\nnon-key: non-value\n\n{TRAILER}\n", ( - # check when trailer has inconsistent whitespace + # Check when trailer has inconsistent whitespace. f"Subject\n \nSome multiline\n body of a function\n\nnon-key: non-value\n\n" f"{KEY_1}:{VALUE_1_1}\n{KEY_2} : {VALUE_2}\n{KEY_1}: {VALUE_1_2}\n" ), @@ -523,7 +525,7 @@ def test_trailers(self): KEY_2: [VALUE_2], } - # check that trailer stays empty for multiple msg combinations + # Check that the trailer stays empty for multiple msg combinations. msgs = [ "Subject\n", "Subject\n\nBody with some\nText\n", @@ -539,7 +541,7 @@ def test_trailers(self): assert commit.trailers_list == [] assert commit.trailers_dict == {} - # check that only the last key value paragraph is evaluated + # Check that only the last key value paragraph is evaluated. commit = copy.copy(self.rorepo.commit("master")) commit.message = f"Subject\n\nMultiline\nBody\n\n{KEY_1}: {VALUE_1_1}\n\n{KEY_2}: {VALUE_2}\n" assert commit.trailers_list == [(KEY_2, VALUE_2)] diff --git a/test/test_config.py b/test/test_config.py index f805570d5..2e278bec2 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -50,25 +50,25 @@ def test_read_write(self): for filename in ("git_config", "git_config_global"): file_obj = self._to_memcache(fixture_path(filename)) with GitConfigParser(file_obj, read_only=False) as w_config: - w_config.read() # enforce reading + w_config.read() # Enforce reading. assert w_config._sections - w_config.write() # enforce writing + w_config.write() # Enforce writing. - # we stripped lines when reading, so the results differ + # We stripped lines when reading, so the results differ. assert file_obj.getvalue() self.assertEqual( file_obj.getvalue(), self._to_memcache(fixture_path(filename)).getvalue(), ) - # creating an additional config writer must fail due to exclusive access + # Creating an additional config writer must fail due to exclusive access. with self.assertRaises(IOError): GitConfigParser(file_obj, read_only=False) - # should still have a lock and be able to make changes + # Should still have a lock and be able to make changes. assert w_config._lock._has_lock() - # changes should be written right away + # Changes should be written right away. sname = "my_section" oname = "mykey" val = "myvalue" @@ -93,11 +93,11 @@ def test_read_write(self): def test_includes_order(self): with GitConfigParser(list(map(fixture_path, ("git_config", "git_config_global")))) as r_config: - r_config.read() # enforce reading - # Simple inclusions, again checking them taking precedence + r_config.read() # Enforce reading. + # Simple inclusions, again checking them taking precedence. assert r_config.get_value("sec", "var0") == "value0_included" # This one should take the git_config_global value since included - # values must be considered as soon as they get them + # values must be considered as soon as they get them. assert r_config.get_value("diff", "tool") == "meld" try: # FIXME: Split this assertion out somehow and mark it xfail (or fix it). @@ -111,16 +111,16 @@ def test_lock_reentry(self, rw_dir): gcp = GitConfigParser(fpl, read_only=False) with gcp as cw: cw.set_value("include", "some_value", "a") - # entering again locks the file again... + # Entering again locks the file again... with gcp as cw: cw.set_value("include", "some_other_value", "b") - # ...so creating an additional config writer must fail due to exclusive access + # ...so creating an additional config writer must fail due to exclusive access. with self.assertRaises(IOError): GitConfigParser(fpl, read_only=False) # but work when the lock is removed with GitConfigParser(fpl, read_only=False): assert osp.exists(fpl) - # reentering with an existing lock must fail due to exclusive access + # Reentering with an existing lock must fail due to exclusive access. with self.assertRaises(IOError): gcp.__enter__() @@ -152,7 +152,7 @@ def test_base(self): num_sections = 0 num_options = 0 - # test reader methods + # Test reader methods. assert r_config._is_initialized is False for section in r_config.sections(): num_sections += 1 @@ -165,7 +165,7 @@ def test_base(self): assert "\n" not in option assert "\n" not in val - # writing must fail + # Writing must fail. with self.assertRaises(IOError): r_config.set(section, option, None) with self.assertRaises(IOError): @@ -177,11 +177,11 @@ def test_base(self): assert num_sections and num_options assert r_config._is_initialized is True - # get value which doesn't exist, with default + # Get value which doesn't exist, with default. default = "my default value" assert r_config.get_value("doesnt", "exist", default) == default - # it raises if there is no default though + # It raises if there is no default though. with self.assertRaises(cp.NoSectionError): r_config.get_value("doesnt", "exist") @@ -228,7 +228,7 @@ def check_test_value(cr, value): # end for each test to verify assert len(cr.items("include")) == 8, "Expected all include sections to be merged" - # test writable config writers - assure write-back doesn't involve includes + # Test writable config writers - assure write-back doesn't involve includes. with GitConfigParser(fpa, read_only=False, merge_includes=True) as cw: tv = "x" write_test_value(cw, tv) @@ -237,7 +237,7 @@ def check_test_value(cr, value): with self.assertRaises(cp.NoSectionError): check_test_value(cr, tv) - # But can make it skip includes altogether, and thus allow write-backs + # But can make it skip includes altogether, and thus allow write-backs. with GitConfigParser(fpa, read_only=False, merge_includes=False) as cw: write_test_value(cw, tv) @@ -246,11 +246,11 @@ def check_test_value(cr, value): @with_rw_directory def test_conditional_includes_from_git_dir(self, rw_dir): - # Initiate repository path + # Initiate repository path. git_dir = osp.join(rw_dir, "target1", "repo1") os.makedirs(git_dir) - # Initiate mocked repository + # Initiate mocked repository. repo = mock.Mock(git_dir=git_dir) # Initiate config files. @@ -313,11 +313,11 @@ def test_conditional_includes_from_git_dir(self, rw_dir): @with_rw_directory def test_conditional_includes_from_branch_name(self, rw_dir): - # Initiate mocked branch + # Initiate mocked branch. branch = mock.Mock() type(branch).name = mock.PropertyMock(return_value="/foo/branch") - # Initiate mocked repository + # Initiate mocked repository. repo = mock.Mock(active_branch=branch) # Initiate config files. diff --git a/test/test_db.py b/test/test_db.py index ebf73b535..acf8379db 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -3,6 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + from git.db import GitCmdObjectDB from git.exc import BadObject from test.lib import TestBase @@ -15,12 +16,12 @@ class TestDB(TestBase): def test_base(self): gdb = GitCmdObjectDB(osp.join(self.rorepo.git_dir, "objects"), self.rorepo.git) - # partial to complete - works with everything + # Partial to complete - works with everything. hexsha = bin_to_hex(gdb.partial_to_complete_sha_hex("0.1.6")) assert len(hexsha) == 40 assert bin_to_hex(gdb.partial_to_complete_sha_hex(hexsha[:20])) == hexsha - # fails with BadObject + # Fails with BadObject. for invalid_rev in ("0000", "bad/ref", "super bad"): self.assertRaises(BadObject, gdb.partial_to_complete_sha_hex, invalid_rev) diff --git a/test/test_diff.py b/test/test_diff.py index 5aa4408bf..1d9c398c2 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -4,6 +4,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import ddt import shutil import tempfile @@ -44,7 +45,7 @@ def tearDown(self): shutil.rmtree(self.submodule_dir) def _assert_diff_format(self, diffs): - # verify that the format of the diff is sane + # Verify that the format of the diff is sane. for diff in diffs: if diff.a_mode: assert isinstance(diff.a_mode, int) @@ -60,7 +61,7 @@ def _assert_diff_format(self, diffs): @with_rw_directory def test_diff_with_staged_file(self, rw_dir): - # SETUP INDEX WITH MULTIPLE STAGES + # SET UP INDEX WITH MULTIPLE STAGES r = Repo.init(rw_dir) fp = osp.join(rw_dir, "hello.txt") with open(fp, "w") as fs: @@ -88,11 +89,11 @@ def test_diff_with_staged_file(self, rw_dir): fs.write("Hallo Welt") r.git.commit(all=True, message="change on topic branch") - # there must be a merge-conflict + # There must be a merge conflict. with self.assertRaises(GitCommandError): r.git.cherry_pick("master") - # Now do the actual testing - this should just work + # Now do the actual testing - this should just work. self.assertEqual(len(r.index.diff(None)), 2) self.assertEqual( @@ -255,7 +256,7 @@ def test_diff_initial_commit(self): self.assertIsNotNone(diff_index[0].new_file) self.assertEqual(diff_index[0].diff, "") - # ...and with creating a patch + # ...and with creating a patch. diff_index = initial_commit.diff(NULL_TREE, create_patch=True) self.assertIsNone(diff_index[0].a_path, repr(diff_index[0].a_path)) self.assertEqual(diff_index[0].b_path, "CHANGES", repr(diff_index[0].b_path)) @@ -292,8 +293,8 @@ def test_diff_unsafe_paths(self): self.assertEqual(res[13].b_path, 'b/"with even more quotes"') def test_diff_patch_format(self): - # test all of the 'old' format diffs for completeness - it should at least - # be able to deal with it + # Test all of the 'old' format diffs for completeness - it should at least + # be able to deal with it. fixtures = ( "diff_2", "diff_2f", @@ -321,14 +322,14 @@ def test_diff_with_spaces(self): def test_diff_submodule(self): """Test that diff is able to correctly diff commits that cover submodule changes""" - # Init a temp git repo that will be referenced as a submodule + # Init a temp git repo that will be referenced as a submodule. sub = Repo.init(self.submodule_dir) with open(self.submodule_dir + "/subfile", "w") as sub_subfile: sub_subfile.write("") sub.index.add(["subfile"]) sub.index.commit("first commit") - # Init a temp git repo that will incorporate the submodule + # Init a temp git repo that will incorporate the submodule. repo = Repo.init(self.repo_dir) with open(self.repo_dir + "/test", "w") as foo_test: foo_test.write("") @@ -337,7 +338,7 @@ def test_diff_submodule(self): repo.index.commit("first commit") repo.create_tag("1") - # Add a commit to the submodule + # Add a commit to the submodule. submodule = repo.submodule("subtest") with open(self.repo_dir + "/sub/subfile", "w") as foo_sub_subfile: foo_sub_subfile.write("blub") @@ -345,19 +346,19 @@ def test_diff_submodule(self): submodule.module().index.commit("changed subfile") submodule.binsha = submodule.module().head.commit.binsha - # Commit submodule updates in parent repo + # Commit submodule updates in parent repo. repo.index.add([submodule]) repo.index.commit("submodule changed") repo.create_tag("2") diff = repo.commit("1").diff(repo.commit("2"))[0] # If diff is unable to find the commit hashes (looks in wrong repo) the *_blob.size - # property will be a string containing exception text, an int indicates success + # property will be a string containing exception text, an int indicates success. self.assertIsInstance(diff.a_blob.size, int) self.assertIsInstance(diff.b_blob.size, int) def test_diff_interface(self): - # test a few variations of the main diff routine + # Test a few variations of the main diff routine. assertion_map = {} for i, commit in enumerate(self.rorepo.iter_commits("0.1.6", max_count=2)): diff_item = commit @@ -379,7 +380,7 @@ def test_diff_interface(self): assertion_map[key] = assertion_map[key] + len(list(diff_index.iter_change_type(ct))) # END for each changetype - # check entries + # Check entries. diff_set = set() diff_set.add(diff_index[0]) diff_set.add(diff_index[0]) @@ -398,14 +399,14 @@ def test_diff_interface(self): # END for each other side # END for each commit - # assert we could always find at least one instance of the members we + # Assert that we could always find at least one instance of the members we # can iterate in the diff index - if not this indicates its not working correctly - # or our test does not span the whole range of possibilities + # or our test does not span the whole range of possibilities. for key, value in assertion_map.items(): self.assertIsNotNone(value, "Did not find diff for %s" % key) # END for each iteration type - # test path not existing in the index - should be ignored + # Test path not existing in the index - should be ignored. c = self.rorepo.head.commit cp = c.parents[0] diff_index = c.diff(cp, ["does/not/exist"]) @@ -415,7 +416,7 @@ def test_diff_interface(self): def test_rename_override(self, rw_dir): """Test disabling of diff rename detection""" - # create and commit file_a.txt + # Create and commit file_a.txt. repo = Repo.init(rw_dir) file_a = osp.join(rw_dir, "file_a.txt") with open(file_a, "w", encoding="utf-8") as outfile: @@ -423,10 +424,10 @@ def test_rename_override(self, rw_dir): repo.git.add(Git.polish_url(file_a)) repo.git.commit(message="Added file_a.txt") - # remove file_a.txt + # Remove file_a.txt. repo.git.rm(Git.polish_url(file_a)) - # create and commit file_b.txt with similarity index of 52 + # Create and commit file_b.txt with similarity index of 52. file_b = osp.join(rw_dir, "file_b.txt") with open(file_b, "w", encoding="utf-8") as outfile: outfile.write("hello world\nhello world") @@ -436,7 +437,7 @@ def test_rename_override(self, rw_dir): commit_a = repo.commit("HEAD") commit_b = repo.commit("HEAD~1") - # check default diff command with renamed files enabled + # Check default diff command with renamed files enabled. diffs = commit_b.diff(commit_a) self.assertEqual(1, len(diffs)) diff = diffs[0] @@ -444,35 +445,35 @@ def test_rename_override(self, rw_dir): self.assertEqual("file_a.txt", diff.rename_from) self.assertEqual("file_b.txt", diff.rename_to) - # check diff with rename files disabled + # Check diff with rename files disabled. diffs = commit_b.diff(commit_a, no_renames=True) self.assertEqual(2, len(diffs)) - # check fileA.txt deleted + # Check fileA.txt deleted. diff = diffs[0] self.assertEqual(True, diff.deleted_file) self.assertEqual("file_a.txt", diff.a_path) - # check fileB.txt added + # Check fileB.txt added. diff = diffs[1] self.assertEqual(True, diff.new_file) self.assertEqual("file_b.txt", diff.a_path) - # check diff with high similarity index + # Check diff with high similarity index. diffs = commit_b.diff(commit_a, split_single_char_options=False, M="75%") self.assertEqual(2, len(diffs)) - # check fileA.txt deleted + # Check fileA.txt deleted. diff = diffs[0] self.assertEqual(True, diff.deleted_file) self.assertEqual("file_a.txt", diff.a_path) - # check fileB.txt added + # Check fileB.txt added. diff = diffs[1] self.assertEqual(True, diff.new_file) self.assertEqual("file_b.txt", diff.a_path) - # check diff with low similarity index + # Check diff with low similarity index. diffs = commit_b.diff(commit_a, split_single_char_options=False, M="40%") self.assertEqual(1, len(diffs)) diff = diffs[0] diff --git a/test/test_fun.py b/test/test_fun.py index f39955aa0..0015b30c6 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -32,21 +32,21 @@ def _assert_index_entries(self, entries, trees): # END assert entry matches fully def test_aggressive_tree_merge(self): - # head tree with additions, removals and modification compared to its predecessor + # Head tree with additions, removals and modification compared to its predecessor. odb = self.rorepo.odb HC = self.rorepo.commit("6c1faef799095f3990e9970bc2cb10aa0221cf9c") H = HC.tree B = HC.parents[0].tree - # entries from single tree + # Entries from single tree. trees = [H.binsha] self._assert_index_entries(aggressive_tree_merge(odb, trees), trees) - # from multiple trees + # From multiple trees. trees = [B.binsha, H.binsha] self._assert_index_entries(aggressive_tree_merge(odb, trees), trees) - # three way, no conflict + # Three way, no conflict. tree = self.rorepo.tree B = tree("35a09c0534e89b2d43ec4101a5fb54576b577905") H = tree("4fe5cfa0e063a8d51a1eb6f014e2aaa994e5e7d4") @@ -54,18 +54,18 @@ def test_aggressive_tree_merge(self): trees = [B.binsha, H.binsha, M.binsha] self._assert_index_entries(aggressive_tree_merge(odb, trees), trees) - # three-way, conflict in at least one file, both modified + # Three-way, conflict in at least one file, both modified. B = tree("a7a4388eeaa4b6b94192dce67257a34c4a6cbd26") H = tree("f9cec00938d9059882bb8eabdaf2f775943e00e5") M = tree("44a601a068f4f543f73fd9c49e264c931b1e1652") trees = [B.binsha, H.binsha, M.binsha] self._assert_index_entries(aggressive_tree_merge(odb, trees), trees) - # too many trees + # Too many trees. self.assertRaises(ValueError, aggressive_tree_merge, odb, trees * 2) def mktree(self, odb, entries): - """create a tree from the given tree entries and safe it to the database""" + """Create a tree from the given tree entries and safe it to the database.""" sio = BytesIO() tree_to_stream(entries, sio.write) sio.seek(0) @@ -92,84 +92,84 @@ def assert_entries(entries, num_entries, has_conflict=False): odb = rwrepo.odb - # base tree + # Base tree. bfn = "basefile" fbase = mkfile(bfn, shaa) tb = mktree(odb, [fbase]) - # non-conflicting new files, same data + # Non-conflicting new files, same data. fa = mkfile("1", shab) th = mktree(odb, [fbase, fa]) fb = mkfile("2", shac) tm = mktree(odb, [fbase, fb]) - # two new files, same base file + # Two new files, same base file. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 3) - # both delete same file, add own one + # Both delete same file, add own one. fa = mkfile("1", shab) th = mktree(odb, [fa]) fb = mkfile("2", shac) tm = mktree(odb, [fb]) - # two new files + # Two new files. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 2) - # same file added in both, differently + # Same file added in both, differently. fa = mkfile("1", shab) th = mktree(odb, [fa]) fb = mkfile("1", shac) tm = mktree(odb, [fb]) - # expect conflict + # Expect conflict. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 2, True) - # same file added, different mode + # Same file added, different mode. fa = mkfile("1", shab) th = mktree(odb, [fa]) fb = mkcommit("1", shab) tm = mktree(odb, [fb]) - # expect conflict + # Expect conflict. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 2, True) - # same file added in both + # Same file added in both. fa = mkfile("1", shab) th = mktree(odb, [fa]) fb = mkfile("1", shab) tm = mktree(odb, [fb]) - # expect conflict + # Expect conflict. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 1) - # modify same base file, differently + # Modify same base file, differently. fa = mkfile(bfn, shab) th = mktree(odb, [fa]) fb = mkfile(bfn, shac) tm = mktree(odb, [fb]) - # conflict, 3 versions on 3 stages + # Conflict, 3 versions on 3 stages. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 3, True) - # change mode on same base file, by making one a commit, the other executable - # no content change ( this is totally unlikely to happen in the real world ) + # Change mode on same base file, by making one a commit, the other executable, + # no content change (this is totally unlikely to happen in the real world). fa = mkcommit(bfn, shaa) th = mktree(odb, [fa]) fb = mkfile(bfn, shaa, executable=1) tm = mktree(odb, [fb]) - # conflict, 3 versions on 3 stages, because of different mode + # Conflict, 3 versions on 3 stages, because of different mode. trees = [tb, th, tm] assert_entries(aggressive_tree_merge(odb, trees), 3, True) for is_them in range(2): - # only we/they change contents + # Only we/they change contents. fa = mkfile(bfn, shab) th = mktree(odb, [fa]) @@ -179,7 +179,7 @@ def assert_entries(entries, num_entries, has_conflict=False): entries = aggressive_tree_merge(odb, trees) assert len(entries) == 1 and entries[0].binsha == shab - # only we/they change the mode + # Only we/they change the mode. fa = mkcommit(bfn, shaa) th = mktree(odb, [fa]) @@ -189,14 +189,14 @@ def assert_entries(entries, num_entries, has_conflict=False): entries = aggressive_tree_merge(odb, trees) assert len(entries) == 1 and entries[0].binsha == shaa and entries[0].mode == fa[1] - # one side deletes, the other changes = conflict + # One side deletes, the other changes = conflict. fa = mkfile(bfn, shab) th = mktree(odb, [fa]) tm = mktree(odb, []) trees = [tb, th, tm] if is_them: trees = [tb, tm, th] - # as one is deleted, there are only 2 entries + # As one is deleted, there are only 2 entries. assert_entries(aggressive_tree_merge(odb, trees), 2, True) # END handle ours, theirs @@ -227,19 +227,19 @@ def _assert_tree_entries(self, entries, num_trees): assert len(entry) == num_trees paths = {e[2] for e in entry if e} - # only one path per set of entries + # Only one path per set of entries. assert len(paths) == 1 # END verify entry def test_tree_traversal(self): - # low level tree tarversal + # Low level tree traversal. odb = self.rorepo.odb H = self.rorepo.tree("29eb123beb1c55e5db4aa652d843adccbd09ae18") # head tree M = self.rorepo.tree("e14e3f143e7260de9581aee27e5a9b2645db72de") # merge tree B = self.rorepo.tree("f606937a7a21237c866efafcad33675e6539c103") # base tree B_old = self.rorepo.tree("1f66cfbbce58b4b552b041707a12d437cc5f400a") # old base tree - # two very different trees + # Two very different trees. entries = traverse_trees_recursive(odb, [B_old.binsha, H.binsha], "") self._assert_tree_entries(entries, 2) @@ -247,17 +247,17 @@ def test_tree_traversal(self): assert len(oentries) == len(entries) self._assert_tree_entries(oentries, 2) - # single tree + # Single tree. is_no_tree = lambda i, d: i.type != "tree" entries = traverse_trees_recursive(odb, [B.binsha], "") assert len(entries) == len(list(B.traverse(predicate=is_no_tree))) self._assert_tree_entries(entries, 1) - # two trees + # Two trees. entries = traverse_trees_recursive(odb, [B.binsha, H.binsha], "") self._assert_tree_entries(entries, 2) - # tree trees + # Three trees. entries = traverse_trees_recursive(odb, [B.binsha, H.binsha, M.binsha], "") self._assert_tree_entries(entries, 3) @@ -275,7 +275,7 @@ def test_tree_traversal_single(self): @with_rw_directory def test_linked_worktree_traversal(self, rw_dir): - """Check that we can identify a linked worktree based on a .git file""" + """Check that we can identify a linked worktree based on a .git file.""" git = Git(rw_dir) if git.version_info[:3] < (2, 5, 1): raise RuntimeError("worktree feature unsupported (test needs git 2.5.1 or later)") diff --git a/test/test_git.py b/test/test_git.py index cf82d9ac7..aa7a415e5 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -4,6 +4,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import inspect import logging import os @@ -77,14 +78,14 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): self.assertEqual(["--max-count=0"], self.git.transform_kwargs(**{"max_count": 0})) self.assertEqual([], self.git.transform_kwargs(**{"max_count": None})) - # Multiple args are supported by using lists/tuples + # Multiple args are supported by using lists/tuples. self.assertEqual( ["-L", "1-3", "-L", "12-18"], self.git.transform_kwargs(**{"L": ("1-3", "12-18")}), ) self.assertEqual(["-C", "-C"], self.git.transform_kwargs(**{"C": [True, True, None, False]})) - # order is undefined + # Order is undefined. res = self.git.transform_kwargs(**{"s": True, "t": True}) self.assertEqual({"-s", "-t"}, set(res)) @@ -189,7 +190,7 @@ def test_it_accepts_stdin(self): @mock.patch.object(Git, "execute") def test_it_ignores_false_kwargs(self, git): - # this_should_not_be_ignored=False implies it *should* be ignored + # this_should_not_be_ignored=False implies it *should* be ignored. self.git.version(pass_this_kwarg=False) self.assertTrue("pass_this_kwarg" not in git.call_args[1]) @@ -218,31 +219,31 @@ def test_it_accepts_environment_variables(self): self.assertEqual(commit, "4cfd6b0314682d5a58f80be39850bad1640e9241") def test_persistent_cat_file_command(self): - # read header only + # Read header only. hexsha = "b2339455342180c7cc1e9bba3e9f181f7baa5167" g = self.git.cat_file(batch_check=True, istream=subprocess.PIPE, as_process=True) g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info = g.stdout.readline() - # read header + data + # Read header + data. g = self.git.cat_file(batch=True, istream=subprocess.PIPE, as_process=True) g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info_two = g.stdout.readline() self.assertEqual(obj_info, obj_info_two) - # read data - have to read it in one large chunk + # Read data - have to read it in one large chunk. size = int(obj_info.split()[2]) g.stdout.read(size) g.stdout.read(1) - # now we should be able to read a new object + # Now we should be able to read a new object. g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() self.assertEqual(g.stdout.readline(), obj_info) - # same can be achieved using the respective command functions + # Same can be achieved using the respective command functions. hexsha, typename, size = self.git.get_object_header(hexsha) hexsha, typename_two, size_two, _ = self.git.get_object_data(hexsha) self.assertEqual(typename, typename_two) @@ -264,31 +265,31 @@ def test_cmd_override(self): self.assertRaises(GitCommandNotFound, self.git.version) def test_refresh(self): - # test a bad git path refresh + # Test a bad git path refresh. self.assertRaises(GitCommandNotFound, refresh, "yada") - # test a good path refresh + # Test a good path refresh. which_cmd = "where" if is_win else "command -v" path = os.popen("{0} git".format(which_cmd)).read().strip().split("\n")[0] refresh(path) def test_options_are_passed_to_git(self): - # This work because any command after git --version is ignored + # This works because any command after git --version is ignored. git_version = self.git(version=True).NoOp() git_command_version = self.git.version() self.assertEqual(git_version, git_command_version) def test_persistent_options(self): git_command_version = self.git.version() - # analog to test_options_are_passed_to_git + # Analog to test_options_are_passed_to_git. self.git.set_persistent_git_options(version=True) git_version = self.git.NoOp() self.assertEqual(git_version, git_command_version) - # subsequent calls keep this option: + # Subsequent calls keep this option: git_version_2 = self.git.NoOp() self.assertEqual(git_version_2, git_command_version) - # reset to empty: + # Reset to empty: self.git.set_persistent_git_options() self.assertRaises(GitCommandError, self.git.NoOp) @@ -301,7 +302,7 @@ def test_change_to_transform_kwargs_does_not_break_command_options(self): self.git.log(n=1) def test_insert_after_kwarg_raises(self): - # This isn't a complete add command, which doesn't matter here + # This isn't a complete add command, which doesn't matter here. self.assertRaises(ValueError, self.git.remote, "add", insert_kwargs_after="foo") def test_env_vars_passed_to_git(self): @@ -311,10 +312,10 @@ def test_env_vars_passed_to_git(self): @with_rw_directory def test_environment(self, rw_dir): - # sanity check + # Sanity check. self.assertEqual(self.git.environment(), {}) - # make sure the context manager works and cleans up after itself + # Make sure the context manager works and cleans up after itself. with self.git.custom_environment(PWD="/tmp"): self.assertEqual(self.git.environment(), {"PWD": "/tmp"}) diff --git a/test/test_index.py b/test/test_index.py index 06db3aedd..c73d25543 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -93,7 +93,7 @@ def _fprogress_add(self, path, done, item): self._fprogress(path, done, item) def _reset_progress(self): - # maps paths to the count of calls + # Maps paths to the count of calls. self._fprogress_map = {} def _assert_entries(self, entries): @@ -104,12 +104,12 @@ def _assert_entries(self, entries): # END for each entry def test_index_file_base(self): - # read from file + # Read from file. index = IndexFile(self.rorepo, fixture_path("index")) assert index.entries assert index.version > 0 - # test entry + # Test entry. entry = next(iter(index.entries.values())) for attr in ( "path", @@ -128,17 +128,17 @@ def test_index_file_base(self): getattr(entry, attr) # END for each method - # test update + # Test update. entries = index.entries assert isinstance(index.update(), IndexFile) assert entries is not index.entries - # test stage + # Test stage. index_merge = IndexFile(self.rorepo, fixture_path("index_merge")) self.assertEqual(len(index_merge.entries), 106) assert len([e for e in index_merge.entries.values() if e.stage != 0]) - # write the data - it must match the original + # Write the data - it must match the original. tmpfile = tempfile.mktemp() index_merge.write(tmpfile) with open(tmpfile, "rb") as fp: @@ -146,7 +146,7 @@ def test_index_file_base(self): os.remove(tmpfile) def _cmp_tree_index(self, tree, index): - # fail unless both objects contain the same paths and blobs + # Fail unless both objects contain the same paths and blobs. if isinstance(tree, str): tree = self.rorepo.commit(tree).tree @@ -169,14 +169,14 @@ def add_bad_blob(): rw_repo.index.add([Blob(rw_repo, b"f" * 20, "bad-permissions", "foo")]) try: - ## 1st fail on purpose adding into index. + ## First, fail on purpose adding into index. add_bad_blob() except Exception as ex: msg_py3 = "required argument is not an integer" msg_py2 = "cannot convert argument to integer" assert msg_py2 in str(ex) or msg_py3 in str(ex) - ## 2nd time should not fail due to stray lock file + ## The second time should not fail due to stray lock file. try: add_bad_blob() except Exception as ex: @@ -188,17 +188,17 @@ def test_index_file_from_tree(self, rw_repo): cur_sha = "4b43ca7ff72d5f535134241e7c797ddc9c7a3573" other_sha = "39f85c4358b7346fee22169da9cad93901ea9eb9" - # simple index from tree + # Simple index from tree. base_index = IndexFile.from_tree(rw_repo, common_ancestor_sha) assert base_index.entries self._cmp_tree_index(common_ancestor_sha, base_index) - # merge two trees - its like a fast-forward + # Merge two trees - it's like a fast-forward. two_way_index = IndexFile.from_tree(rw_repo, common_ancestor_sha, cur_sha) assert two_way_index.entries self._cmp_tree_index(cur_sha, two_way_index) - # merge three trees - here we have a merge conflict + # Merge three trees - here we have a merge conflict. three_way_index = IndexFile.from_tree(rw_repo, common_ancestor_sha, cur_sha, other_sha) assert len([e for e in three_way_index.entries.values() if e.stage != 0]) @@ -209,19 +209,19 @@ def test_index_file_from_tree(self, rw_repo): assert merge_blobs[0][0] in (1, 2, 3) assert isinstance(merge_blobs[0][1], Blob) - # test BlobFilter + # Test BlobFilter. prefix = "lib/git" for _stage, blob in base_index.iter_blobs(BlobFilter([prefix])): assert blob.path.startswith(prefix) - # writing a tree should fail with an unmerged index + # Writing a tree should fail with an unmerged index. self.assertRaises(UnmergedEntriesError, three_way_index.write_tree) - # removed unmerged entries + # Removed unmerged entries. unmerged_blob_map = three_way_index.unmerged_blobs() assert unmerged_blob_map - # pick the first blob at the first stage we find and use it as resolved version + # Pick the first blob at the first stage we find and use it as resolved version. three_way_index.resolve_blobs(line[0][1] for line in unmerged_blob_map.values()) tree = three_way_index.write_tree() assert isinstance(tree, Tree) @@ -239,13 +239,13 @@ def test_index_merge_tree(self, rw_repo): self.assertEqual(len({self.rorepo, self.rorepo, rw_repo, rw_repo}), 2) # SINGLE TREE MERGE - # current index is at the (virtual) cur_commit + # Current index is at the (virtual) cur_commit. next_commit = "4c39f9da792792d4e73fc3a5effde66576ae128c" parent_commit = rw_repo.head.commit.parents[0] manifest_key = IndexFile.entry_key("MANIFEST.in", 0) manifest_entry = rw_repo.index.entries[manifest_key] rw_repo.index.merge_tree(next_commit) - # only one change should be recorded + # Only one change should be recorded. assert manifest_entry.binsha != rw_repo.index.entries[manifest_key].binsha rw_repo.index.reset(rw_repo.head) @@ -255,40 +255,40 @@ def test_index_merge_tree(self, rw_repo): ############# # Add a change with a NULL sha that should conflict with next_commit. We # pretend there was a change, but we do not even bother adding a proper - # sha for it ( which makes things faster of course ) + # sha for it (which makes things faster of course). manifest_fake_entry = BaseIndexEntry((manifest_entry[0], b"\0" * 20, 0, manifest_entry[3])) - # try write flag + # Try write flag. self._assert_entries(rw_repo.index.add([manifest_fake_entry], write=False)) - # add actually resolves the null-hex-sha for us as a feature, but we can - # edit the index manually + # Add actually resolves the null-hex-sha for us as a feature, but we can + # edit the index manually. assert rw_repo.index.entries[manifest_key].binsha != Object.NULL_BIN_SHA - # must operate on the same index for this ! Its a bit problematic as - # it might confuse people + # We must operate on the same index for this! It's a bit problematic as + # it might confuse people. index = rw_repo.index index.entries[manifest_key] = IndexEntry.from_base(manifest_fake_entry) index.write() self.assertEqual(rw_repo.index.entries[manifest_key].hexsha, Diff.NULL_HEX_SHA) - # write an unchanged index ( just for the fun of it ) + # Write an unchanged index (just for the fun of it). rw_repo.index.write() - # a three way merge would result in a conflict and fails as the command will + # A three way merge would result in a conflict and fails as the command will # not overwrite any entries in our index and hence leave them unmerged. This is - # mainly a protection feature as the current index is not yet in a tree + # mainly a protection feature as the current index is not yet in a tree. self.assertRaises(GitCommandError, index.merge_tree, next_commit, base=parent_commit) - # the only way to get the merged entries is to safe the current index away into a tree, - # which is like a temporary commit for us. This fails as well as the NULL sha deos not - # have a corresponding object - # NOTE: missing_ok is not a kwarg anymore, missing_ok is always true + # The only way to get the merged entries is to safe the current index away into a tree, + # which is like a temporary commit for us. This fails as well as the NULL sha does not + # have a corresponding object. + # NOTE: missing_ok is not a kwarg anymore, missing_ok is always true. # self.assertRaises(GitCommandError, index.write_tree) - # if missing objects are okay, this would work though ( they are always okay now ) - # As we can't read back the tree with NULL_SHA, we rather set it to something else + # If missing objects are okay, this would work though (they are always okay now). + # As we can't read back the tree with NULL_SHA, we rather set it to something else. index.entries[manifest_key] = IndexEntry(manifest_entry[:1] + (hex_to_bin("f" * 40),) + manifest_entry[2:]) tree = index.write_tree() - # now make a proper three way merge with unmerged entries + # Now make a proper three way merge with unmerged entries. unmerged_tree = IndexFile.from_tree(rw_repo, parent_commit, tree, next_commit) unmerged_blobs = unmerged_tree.unmerged_blobs() self.assertEqual(len(unmerged_blobs), 1) @@ -296,49 +296,49 @@ def test_index_merge_tree(self, rw_repo): @with_rw_repo("0.1.6") def test_index_file_diffing(self, rw_repo): - # default Index instance points to our index + # Default Index instance points to our index. index = IndexFile(rw_repo) assert index.path is not None assert len(index.entries) - # write the file back + # Write the file back. index.write() - # could sha it, or check stats + # Could sha it, or check stats. - # test diff - # resetting the head will leave the index in a different state, and the - # diff will yield a few changes + # Test diff. + # Resetting the head will leave the index in a different state, and the + # diff will yield a few changes. cur_head_commit = rw_repo.head.reference.commit rw_repo.head.reset("HEAD~6", index=True, working_tree=False) - # diff against same index is 0 + # Diff against same index is 0. diff = index.diff() self.assertEqual(len(diff), 0) - # against HEAD as string, must be the same as it matches index + # Against HEAD as string, must be the same as it matches index. diff = index.diff("HEAD") self.assertEqual(len(diff), 0) - # against previous head, there must be a difference + # Against previous head, there must be a difference. diff = index.diff(cur_head_commit) assert len(diff) - # we reverse the result + # We reverse the result. adiff = index.diff(str(cur_head_commit), R=True) - odiff = index.diff(cur_head_commit, R=False) # now its not reversed anymore + odiff = index.diff(cur_head_commit, R=False) # Now its not reversed anymore. assert adiff != odiff - self.assertEqual(odiff, diff) # both unreversed diffs against HEAD + self.assertEqual(odiff, diff) # Both unreversed diffs against HEAD. - # against working copy - its still at cur_commit + # Against working copy - it's still at cur_commit. wdiff = index.diff(None) assert wdiff != adiff assert wdiff != odiff - # against something unusual + # Against something unusual. self.assertRaises(ValueError, index.diff, int) - # adjust the index to match an old revision + # Adjust the index to match an old revision. cur_branch = rw_repo.active_branch cur_commit = cur_branch.commit rev_head_parent = "HEAD~1" @@ -347,10 +347,10 @@ def test_index_file_diffing(self, rw_repo): self.assertEqual(cur_branch, rw_repo.active_branch) self.assertEqual(cur_commit, rw_repo.head.commit) - # there must be differences towards the working tree which is in the 'future' + # There must be differences towards the working tree which is in the 'future'. assert index.diff(None) - # reset the working copy as well to current head,to pull 'back' as well + # Reset the working copy as well to current head, to pull 'back' as well. new_data = b"will be reverted" file_path = osp.join(rw_repo.working_tree_dir, "CHANGES") with open(file_path, "wb") as fp: @@ -362,7 +362,7 @@ def test_index_file_diffing(self, rw_repo): with open(file_path, "rb") as fp: assert fp.read() != new_data - # test full checkout + # Test full checkout. test_file = osp.join(rw_repo.working_tree_dir, "CHANGES") with open(test_file, "ab") as fd: fd.write(b"some data") @@ -377,25 +377,25 @@ def test_index_file_diffing(self, rw_repo): self._assert_fprogress([None]) assert osp.isfile(test_file) - # individual file + # Individual file. os.remove(test_file) rval = index.checkout(test_file, fprogress=self._fprogress) self.assertEqual(list(rval)[0], "CHANGES") self._assert_fprogress([test_file]) assert osp.exists(test_file) - # checking out non-existing file throws + # Checking out non-existing file throws. self.assertRaises(CheckoutError, index.checkout, "doesnt_exist_ever.txt.that") self.assertRaises(CheckoutError, index.checkout, paths=["doesnt/exist"]) - # checkout file with modifications + # Check out file with modifications. append_data = b"hello" with open(test_file, "ab") as fp: fp.write(append_data) try: index.checkout(test_file) except CheckoutError as e: - # detailed exceptions are only possible in older git versions + # Detailed exceptions are only possible in older git versions. if rw_repo.git._version_info < (2, 29): self.assertEqual(len(e.failed_files), 1) self.assertEqual(e.failed_files[0], osp.basename(test_file)) @@ -408,19 +408,17 @@ def test_index_file_diffing(self, rw_repo): else: raise AssertionError("Exception CheckoutError not thrown") - # if we force it it should work + # If we force it, it should work. index.checkout(test_file, force=True) assert not open(test_file, "rb").read().endswith(append_data) - # checkout directory + # Check out directory. rmtree(osp.join(rw_repo.working_tree_dir, "lib")) rval = index.checkout("lib") assert len(list(rval)) > 1 def _count_existing(self, repo, files): - """ - Returns count of files that actually exist in the repository directory. - """ + """Return count of files that actually exist in the repository directory.""" existing = 0 basedir = repo.working_tree_dir for f in files: @@ -443,8 +441,8 @@ def test_index_mutation(self, rw_repo): writer.set_value("user", "email", umail) self.assertEqual(writer.get_value("user", "name"), uname) - # remove all of the files, provide a wild mix of paths, BaseIndexEntries, - # IndexEntries + # Remove all of the files, provide a wild mix of paths, BaseIndexEntries, + # IndexEntries. def mixed_iterator(): count = 0 for entry in index.entries.values(): @@ -468,29 +466,29 @@ def mixed_iterator(): self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) self.assertEqual(len(index.entries), 0) - # reset the index to undo our changes + # Reset the index to undo our changes. index.reset() self.assertEqual(len(index.entries), num_entries) - # remove with working copy + # Remove with working copy. deleted_files = index.remove(mixed_iterator(), working_tree=True) assert deleted_files self.assertEqual(self._count_existing(rw_repo, deleted_files), 0) - # reset everything + # Reset everything. index.reset(working_tree=True) self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) - # invalid type + # Invalid type. self.assertRaises(TypeError, index.remove, [1]) - # absolute path + # Absolute path. deleted_files = index.remove([osp.join(rw_repo.working_tree_dir, "lib")], r=True) assert len(deleted_files) > 1 self.assertRaises(ValueError, index.remove, ["/doesnt/exists"]) # TEST COMMITTING - # commit changed index + # Commit changed index. cur_commit = cur_head.commit commit_message = "commit default head by Frèderic Çaufl€" @@ -505,7 +503,7 @@ def mixed_iterator(): self.assertEqual(len(new_commit.parents), 1) self.assertEqual(cur_head.commit, cur_commit) - # commit with other actor + # Commit with other actor. cur_commit = cur_head.commit my_author = Actor("Frèderic Çaufl€", "author@example.com") @@ -522,7 +520,7 @@ def mixed_iterator(): self.assertEqual(cur_head.commit, commit_actor) self.assertEqual(cur_head.log()[-1].actor, my_committer) - # commit with author_date and commit_date + # Commit with author_date and commit_date. cur_commit = cur_head.commit commit_message = "commit with dates by Avinash Sajjanshetty" @@ -537,14 +535,14 @@ def mixed_iterator(): self.assertEqual(new_commit.authored_date, 1144447993) self.assertEqual(new_commit.committed_date, 1112911993) - # same index, no parents + # Same index, no parents. commit_message = "index without parents" commit_no_parents = index.commit(commit_message, parent_commits=[], head=True) self.assertEqual(commit_no_parents.message, commit_message) self.assertEqual(len(commit_no_parents.parents), 0) self.assertEqual(cur_head.commit, commit_no_parents) - # same index, multiple parents + # same index, multiple parents. commit_message = "Index with multiple parents\n commit with another line" commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) self.assertEqual(commit_multi_parent.message, commit_message) @@ -553,41 +551,41 @@ def mixed_iterator(): self.assertEqual(commit_multi_parent.parents[1], new_commit) self.assertEqual(cur_head.commit, commit_multi_parent) - # re-add all files in lib - # get the lib folder back on disk, but get an index without it + # Re-add all files in lib. + # Get the lib folder back on disk, but get an index without it. index.reset(new_commit.parents[0], working_tree=True).reset(new_commit, working_tree=False) lib_file_path = osp.join("lib", "git", "__init__.py") assert (lib_file_path, 0) not in index.entries assert osp.isfile(osp.join(rw_repo.working_tree_dir, lib_file_path)) - # directory + # Directory. entries = index.add(["lib"], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) assert len(entries) > 1 - # glob + # Glob. entries = index.reset(new_commit).add([osp.join("lib", "git", "*.py")], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) self.assertEqual(len(entries), 14) - # same file + # Same file. entries = index.reset(new_commit).add( [osp.join(rw_repo.working_tree_dir, "lib", "git", "head.py")] * 2, fprogress=self._fprogress_add, ) self._assert_entries(entries) self.assertEqual(entries[0].mode & 0o644, 0o644) - # would fail, test is too primitive to handle this case + # Would fail, test is too primitive to handle this case. # self._assert_fprogress(entries) self._reset_progress() self.assertEqual(len(entries), 2) - # missing path + # Missing path. self.assertRaises(OSError, index.reset(new_commit).add, ["doesnt/exist/must/raise"]) - # blob from older revision overrides current index revision + # Blob from older revision overrides current index revision. old_blob = new_commit.parents[0].tree.blobs[0] entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) self._assert_entries(entries) @@ -595,7 +593,7 @@ def mixed_iterator(): self.assertEqual(index.entries[(old_blob.path, 0)].hexsha, old_blob.hexsha) self.assertEqual(len(entries), 1) - # mode 0 not allowed + # Mode 0 not allowed. null_hex_sha = Diff.NULL_HEX_SHA null_bin_sha = b"\0" * 20 self.assertRaises( @@ -604,7 +602,7 @@ def mixed_iterator(): [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))], ) - # add new file + # Add new file. new_file_relapath = "my_new_file" self._make_file(new_file_relapath, "hello world", rw_repo) entries = index.reset(new_commit).add( @@ -616,7 +614,7 @@ def mixed_iterator(): self.assertEqual(len(entries), 1) self.assertNotEqual(entries[0].hexsha, null_hex_sha) - # add symlink + # Add symlink. if not is_win: for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): basename = "my_real_symlink" @@ -630,7 +628,7 @@ def mixed_iterator(): self.assertTrue(S_ISLNK(entries[0].mode)) self.assertTrue(S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode)) - # we expect only the target to be written + # We expect only the target to be written. self.assertEqual( index.repo.odb.stream(entries[0].binsha).read().decode("ascii"), target, @@ -640,7 +638,7 @@ def mixed_iterator(): # end for each target # END real symlink test - # add fake symlink and assure it checks-our as symlink + # Add fake symlink and assure it checks-our as symlink. fake_symlink_relapath = "my_fake_symlink" link_target = "/etc/that" fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) @@ -652,7 +650,7 @@ def mixed_iterator(): self.assertEqual(len(entries), 1) self.assertTrue(S_ISLNK(entries[0].mode)) - # assure this also works with an alternate method + # Check that this also works with an alternate method. full_index_entry = IndexEntry.from_base(BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path))) entry_key = index.entry_key(full_index_entry) index.reset(new_commit) @@ -660,24 +658,24 @@ def mixed_iterator(): assert entry_key not in index.entries index.entries[entry_key] = full_index_entry index.write() - index.update() # force reread of entries + index.update() # Force reread of entries. new_entry = index.entries[entry_key] assert S_ISLNK(new_entry.mode) - # a tree created from this should contain the symlink + # A tree created from this should contain the symlink. tree = index.write_tree() assert fake_symlink_relapath in tree - index.write() # flush our changes for the checkout + index.write() # Flush our changes for the checkout. - # checkout the fakelink, should be a link then + # Check out the fake link, should be a link then. assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE]) os.remove(fake_symlink_path) index.checkout(fake_symlink_path) - # on windows we will never get symlinks + # On Windows, we will never get symlinks. if is_win: - # simlinks should contain the link as text ( which is what a - # symlink actually is ) + # Symlinks should contain the link as text (which is what a + # symlink actually is). with open(fake_symlink_path, "rt") as fd: self.assertEqual(fd.read(), link_target) else: @@ -692,24 +690,24 @@ def assert_mv_rval(rval): # END move assertion utility self.assertRaises(ValueError, index.move, ["just_one_path"]) - # file onto existing file + # Try to move a file onto an existing file. files = ["AUTHORS", "LICENSE"] self.assertRaises(GitCommandError, index.move, files) - # again, with force + # Again, with force. assert_mv_rval(index.move(files, f=True)) - # files into directory - dry run + # Move files into a directory - dry run. paths = ["LICENSE", "VERSION", "doc"] rval = index.move(paths, dry_run=True) self.assertEqual(len(rval), 2) assert osp.exists(paths[0]) - # again, no dry run + # Again, no dry run. rval = index.move(paths) assert_mv_rval(rval) - # dir into dir + # Move dir into dir. rval = index.move(["doc", "test"]) assert_mv_rval(rval) @@ -725,7 +723,7 @@ def rewriter(entry): # END rewriter def make_paths(): - # two existing ones, one new one + """Help out the test by yielding two existing paths and one new path.""" yield "CHANGES" yield "ez_setup.py" yield index.entries[index.entry_key("README", 0)] @@ -766,12 +764,12 @@ def make_paths(): for fkey in keys: assert fkey in index.entries - # just the index + # Just the index. index.reset(paths=(arela, afile)) assert akey not in index.entries assert bkey in index.entries - # now with working tree - files on disk as well as entries must be recreated + # Now with working tree - files on disk as well as entries must be recreated. rw_repo.head.commit = nc for absfile in absfiles: os.remove(absfile) @@ -785,8 +783,8 @@ def make_paths(): @with_rw_repo("HEAD") def test_compare_write_tree(self, rw_repo): - # write all trees and compare them - # its important to have a few submodules in there too + """Test writing all trees, comparing them for equality.""" + # It's important to have a few submodules in there too. max_count = 25 count = 0 for commit in rw_repo.head.commit.traverse(): @@ -847,7 +845,7 @@ def test_index_bare_add(self, rw_bare_repo): @with_rw_directory def test_add_utf8P_path(self, rw_dir): - # NOTE: fp is not a Unicode object in python 2 (which is the source of the problem) + # NOTE: fp is not a Unicode object in Python 2 (which is the source of the problem). fp = osp.join(rw_dir, "ø.txt") with open(fp, "wb") as fs: fs.write("content of ø".encode("utf-8")) @@ -858,7 +856,7 @@ def test_add_utf8P_path(self, rw_dir): @with_rw_directory def test_add_a_file_with_wildcard_chars(self, rw_dir): - # see issue #407 + # See issue #407. fp = osp.join(rw_dir, "[.exe") with open(fp, "wb") as f: f.write(b"something") diff --git a/test/test_reflog.py b/test/test_reflog.py index e899ac48c..d5173d2f4 100644 --- a/test/test_reflog.py +++ b/test/test_reflog.py @@ -26,7 +26,7 @@ def test_reflogentry(self): assert e.time[1] == 1 assert e.message == msg - # check representation (roughly) + # Check representation (roughly). assert repr(e).startswith(nullhexsha) def test_base(self): @@ -38,27 +38,27 @@ def test_base(self): rlp_master_ro = RefLog.path(self.rorepo.head) assert osp.isfile(rlp_master_ro) - # simple read + # Simple read. reflog = RefLog.from_file(rlp_master_ro) assert reflog._path is not None assert isinstance(reflog, RefLog) assert len(reflog) - # iter_entries works with path and with stream + # iter_entries works with path and with stream. assert len(list(RefLog.iter_entries(open(rlp_master, "rb")))) assert len(list(RefLog.iter_entries(rlp_master))) - # raise on invalid revlog - # TODO: Try multiple corrupted ones ! + # Raise on invalid revlog. + # TODO: Try multiple corrupted ones! pp = "reflog_invalid_" for suffix in ("oldsha", "newsha", "email", "date", "sep"): self.assertRaises(ValueError, RefLog.from_file, fixture_path(pp + suffix)) # END for each invalid file - # cannot write an uninitialized reflog + # Cannot write an uninitialized reflog. self.assertRaises(ValueError, RefLog().write) - # test serialize and deserialize - results must match exactly + # Test serialize and deserialize - results must match exactly. binsha = hex_to_bin(("f" * 40).encode("ascii")) msg = "my reflog message" cr = self.rorepo.config_reader() @@ -68,33 +68,33 @@ def test_base(self): reflog.to_file(tfile) assert reflog.write() is reflog - # parsed result must match ... + # Parsed result must match... treflog = RefLog.from_file(tfile) assert treflog == reflog - # ... as well as each bytes of the written stream + # ...as well as each bytes of the written stream. assert open(tfile).read() == open(rlp).read() - # append an entry + # Append an entry. entry = RefLog.append_entry(cr, tfile, IndexObject.NULL_BIN_SHA, binsha, msg) assert entry.oldhexsha == IndexObject.NULL_HEX_SHA assert entry.newhexsha == "f" * 40 assert entry.message == msg assert RefLog.from_file(tfile)[-1] == entry - # index entry - # raises on invalid index + # Index entry. + # Raises on invalid index. self.assertRaises(IndexError, RefLog.entry_at, rlp, 10000) - # indices can be positive ... + # Indices can be positive... assert isinstance(RefLog.entry_at(rlp, 0), RefLogEntry) RefLog.entry_at(rlp, 23) - # ... and negative + # ...and negative. for idx in (-1, -24): RefLog.entry_at(rlp, idx) # END for each index to read # END for each reflog - # finally remove our temporary data + # Finally remove our temporary data. rmtree(tdir) diff --git a/test/test_refs.py b/test/test_refs.py index f9fc8b0ad..ae07ce421 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -30,7 +30,7 @@ class TestRefs(TestBase): def test_from_path(self): - # should be able to create any reference directly + # Should be able to create any reference directly. for ref_type in (Reference, Head, TagReference, RemoteReference): for name in ("rela_name", "path/rela_name"): full_path = ref_type.to_full_path(name) @@ -39,9 +39,9 @@ def test_from_path(self): # END for each name # END for each type - # invalid path + # Invalid path. self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") - # works without path check + # Works without path check. TagReference(self.rorepo, "refs/invalid/tag", check_path=False) def test_tag_base(self): @@ -53,7 +53,7 @@ def test_tag_base(self): if tag.tag is not None: tag_object_refs.append(tag) tagobj = tag.tag - # have no dict + # Have no dict. self.assertRaises(AttributeError, setattr, tagobj, "someattr", 1) assert isinstance(tagobj, TagObject) assert tagobj.tag == tag.name @@ -62,7 +62,7 @@ def test_tag_base(self): assert isinstance(tagobj.tagger_tz_offset, int) assert tagobj.message assert tag.object == tagobj - # can't assign the object + # Can't assign the object. self.assertRaises(AttributeError, setattr, tag, "object", tagobj) # END if we have a tag object # END for tag in repo-tags @@ -77,7 +77,7 @@ def test_tags_author(self): assert tagger_name == "Michael Trier" def test_tags(self): - # tag refs can point to tag objects or to commits + # Tag refs can point to tag objects or to commits. s = set() ref_count = 0 for ref in chain(self.rorepo.tags, self.rorepo.heads): @@ -100,8 +100,8 @@ def test_heads(self, rwrepo): assert "refs/heads" in head.path prev_object = head.object cur_object = head.object - assert prev_object == cur_object # represent the same git object - assert prev_object is not cur_object # but are different instances + assert prev_object == cur_object # Represent the same git object... + assert prev_object is not cur_object # ...but are different instances. with head.config_writer() as writer: tv = "testopt" @@ -111,7 +111,7 @@ def test_heads(self, rwrepo): with head.config_writer() as writer: writer.remove_option(tv) - # after the clone, we might still have a tracking branch setup + # After the clone, we might still have a tracking branch setup. head.set_tracking_branch(None) assert head.tracking_branch() is None remote_ref = rwrepo.remotes[0].refs[0] @@ -123,7 +123,7 @@ def test_heads(self, rwrepo): special_name = "feature#123" special_name_remote_ref = SymbolicReference.create(rwrepo, "refs/remotes/origin/%s" % special_name) gp_tracking_branch = rwrepo.create_head("gp_tracking#123") - special_name_remote_ref = rwrepo.remotes[0].refs[special_name] # get correct type + special_name_remote_ref = rwrepo.remotes[0].refs[special_name] # Get correct type. gp_tracking_branch.set_tracking_branch(special_name_remote_ref) TBranch = gp_tracking_branch.tracking_branch() if TBranch is not None: @@ -136,7 +136,7 @@ def test_heads(self, rwrepo): assert TBranch.name == special_name_remote_ref.name # END for each head - # verify REFLOG gets altered + # Verify REFLOG gets altered. head = rwrepo.head cur_head = head.ref cur_commit = cur_head.commit @@ -144,32 +144,31 @@ def test_heads(self, rwrepo): hlog_len = len(head.log()) blog_len = len(cur_head.log()) assert head.set_reference(pcommit, "detached head") is head - # one new log-entry + # One new log-entry. thlog = head.log() assert len(thlog) == hlog_len + 1 assert thlog[-1].oldhexsha == cur_commit.hexsha assert thlog[-1].newhexsha == pcommit.hexsha - # the ref didn't change though + # The ref didn't change though. assert len(cur_head.log()) == blog_len - # head changes once again, cur_head doesn't change + # head changes once again, cur_head doesn't change. head.set_reference(cur_head, "reattach head") assert len(head.log()) == hlog_len + 2 assert len(cur_head.log()) == blog_len - # adjusting the head-ref also adjust the head, so both reflogs are - # altered + # Adjusting the head-ref also adjust the head, so both reflogs are altered. cur_head.set_commit(pcommit, "changing commit") assert len(cur_head.log()) == blog_len + 1 assert len(head.log()) == hlog_len + 3 - # with automatic dereferencing + # With automatic dereferencing. assert head.set_commit(cur_commit, "change commit once again") is head assert len(head.log()) == hlog_len + 4 assert len(cur_head.log()) == blog_len + 2 - # a new branch has just a single entry + # A new branch has just a single entry. other_head = Head.create(rwrepo, "mynewhead", pcommit, logmsg="new head created") log = other_head.log() assert len(log) == 1 @@ -178,7 +177,7 @@ def test_heads(self, rwrepo): @with_rw_repo("HEAD", bare=False) def test_set_tracking_branch_with_import(self, rwrepo): - # prepare included config file + # Prepare included config file. included_config = osp.join(rwrepo.git_dir, "config.include") with GitConfigParser(included_config, read_only=False) as writer: writer.set_value("test", "value", "test") @@ -230,11 +229,11 @@ def test_head_reset(self, rw_repo): cur_head.reset(new_head_commit, index=True, working_tree=True) # index + wt assert cur_head.reference.commit == new_head_commit - # paths - make sure we have something to do + # Paths - make sure we have something to do. rw_repo.index.reset(old_head_commit.parents[0]) cur_head.reset(cur_head, paths="test") cur_head.reset(new_head_commit, paths="lib") - # hard resets with paths don't work, its all or nothing + # Hard resets with paths don't work; it's all or nothing. self.assertRaises( GitCommandError, cur_head.reset, @@ -243,12 +242,12 @@ def test_head_reset(self, rw_repo): paths="lib", ) - # we can do a mixed reset, and then checkout from the index though + # We can do a mixed reset, and then checkout from the index though. cur_head.reset(new_head_commit) rw_repo.index.checkout(["lib"], force=True) - # now that we have a write write repo, change the HEAD reference - its - # like git-reset --soft + # Now that we have a write write repo, change the HEAD reference - it's + # like "git-reset --soft". heads = rw_repo.heads assert heads for head in heads: @@ -259,7 +258,7 @@ def test_head_reset(self, rw_repo): assert not cur_head.is_detached # END for each head - # detach + # Detach. active_head = heads[0] curhead_commit = active_head.commit cur_head.reference = curhead_commit @@ -267,20 +266,20 @@ def test_head_reset(self, rw_repo): assert cur_head.is_detached self.assertRaises(TypeError, getattr, cur_head, "reference") - # tags are references, hence we can point to them + # Tags are references, hence we can point to them. some_tag = rw_repo.tags[0] cur_head.reference = some_tag assert not cur_head.is_detached assert cur_head.commit == some_tag.commit assert isinstance(cur_head.reference, TagReference) - # put HEAD back to a real head, otherwise everything else fails + # Put HEAD back to a real head, otherwise everything else fails. cur_head.reference = active_head - # type check + # Type check. self.assertRaises(ValueError, setattr, cur_head, "reference", "that") - # head handling + # Head handling. commit = "HEAD" prev_head_commit = cur_head.commit for count, new_name in enumerate(("my_new_head", "feature/feature1")): @@ -289,13 +288,13 @@ def test_head_reset(self, rw_repo): assert new_head.is_detached assert cur_head.commit == prev_head_commit assert isinstance(new_head, Head) - # already exists, but has the same value, so its fine + # Already exists, but has the same value, so it's fine. Head.create(rw_repo, new_name, new_head.commit) - # its not fine with a different value + # It's not fine with a different value. self.assertRaises(OSError, Head.create, rw_repo, new_name, new_head.commit.parents[0]) - # force it + # Force it. new_head = Head.create(rw_repo, new_name, actual_commit, force=True) old_path = new_head.path old_name = new_head.name @@ -304,7 +303,7 @@ def test_head_reset(self, rw_repo): assert new_head.rename("hello/world").name == "hello/world" assert new_head.rename(old_name).name == old_name and new_head.path == old_path - # rename with force + # Rename with force. tmp_head = Head.create(rw_repo, "tmphead") self.assertRaises(GitCommandError, tmp_head.rename, new_head) tmp_head.rename(new_head, force=True) @@ -313,15 +312,15 @@ def test_head_reset(self, rw_repo): logfile = RefLog.path(tmp_head) assert osp.isfile(logfile) Head.delete(rw_repo, tmp_head) - # deletion removes the log as well + # Deletion removes the log as well. assert not osp.isfile(logfile) heads = rw_repo.heads assert tmp_head not in heads and new_head not in heads - # force on deletion testing would be missing here, code looks okay though ;) + # Force on deletion testing would be missing here, code looks okay though. ;) # END for each new head name self.assertRaises(TypeError, RemoteReference.create, rw_repo, "some_name") - # tag ref + # Tag ref. tag_name = "5.0.2" TagReference.create(rw_repo, tag_name) self.assertRaises(GitCommandError, TagReference.create, rw_repo, tag_name) @@ -331,7 +330,7 @@ def test_head_reset(self, rw_repo): assert light_tag.commit == cur_head.commit.parents[0] assert light_tag.tag is None - # tag with tag object + # Tag with tag object. other_tag_name = "releases/1.0.2RC" msg = "my mighty tag\nsecond line" obj_tag = TagReference.create(rw_repo, other_tag_name, message=msg) @@ -344,7 +343,7 @@ def test_head_reset(self, rw_repo): tags = rw_repo.tags assert light_tag not in tags and obj_tag not in tags - # remote deletion + # Remote deletion. remote_refs_so_far = 0 remotes = rw_repo.remotes assert remotes @@ -367,11 +366,11 @@ def test_head_reset(self, rw_repo): assert remote_refs_so_far for remote in remotes: - # remotes without references should produce an empty list + # Remotes without references should produce an empty list. self.assertEqual(remote.refs, []) # END for each remote - # change where the active head points to + # Change where the active head points to. if cur_head.is_detached: cur_head.reference = rw_repo.heads[0] @@ -382,17 +381,17 @@ def test_head_reset(self, rw_repo): assert head.commit == cur_head.commit head.commit = old_commit - # setting a non-commit as commit fails, but succeeds as object + # Setting a non-commit as commit fails, but succeeds as object. head_tree = head.commit.tree self.assertRaises(ValueError, setattr, head, "commit", head_tree) assert head.commit == old_commit # and the ref did not change - # we allow heads to point to any object + # We allow heads to point to any object. head.object = head_tree assert head.object == head_tree - # cannot query tree as commit + # Cannot query tree as commit. self.assertRaises(TypeError, getattr, head, "commit") - # set the commit directly using the head. This would never detach the head + # Set the commit directly using the head. This would never detach the head. assert not cur_head.is_detached head.object = old_commit cur_head.reference = head.commit @@ -408,30 +407,30 @@ def test_head_reset(self, rw_repo): assert not cur_head.is_detached assert head.commit == parent_commit - # test checkout + # Test checkout. active_branch = rw_repo.active_branch for head in rw_repo.heads: checked_out_head = head.checkout() assert checked_out_head == head # END for each head to checkout - # checkout with branch creation + # Check out with branch creation. new_head = active_branch.checkout(b="new_head") assert active_branch != rw_repo.active_branch assert new_head == rw_repo.active_branch - # checkout with force as we have a changed a file - # clear file + # Checkout with force as we have a changed a file. + # Clear file. open(new_head.commit.tree.blobs[-1].abspath, "w").close() assert len(new_head.commit.diff(None)) - # create a new branch that is likely to touch the file we changed + # Create a new branch that is likely to touch the file we changed. far_away_head = rw_repo.create_head("far_head", "HEAD~100") self.assertRaises(GitCommandError, far_away_head.checkout) assert active_branch == active_branch.checkout(force=True) assert rw_repo.head.reference != far_away_head - # test reference creation + # Test reference creation. partial_ref = "sub/ref" full_ref = "refs/%s" % partial_ref ref = Reference.create(rw_repo, partial_ref) @@ -439,21 +438,21 @@ def test_head_reset(self, rw_repo): assert ref.object == rw_repo.head.commit self.assertRaises(OSError, Reference.create, rw_repo, full_ref, "HEAD~20") - # it works if it is at the same spot though and points to the same reference + # It works if it is at the same spot though and points to the same reference. assert Reference.create(rw_repo, full_ref, "HEAD").path == full_ref Reference.delete(rw_repo, full_ref) - # recreate the reference using a full_ref + # Recreate the reference using a full_ref. ref = Reference.create(rw_repo, full_ref) assert ref.path == full_ref assert ref.object == rw_repo.head.commit - # recreate using force + # Recreate using force. ref = Reference.create(rw_repo, partial_ref, "HEAD~1", force=True) assert ref.path == full_ref assert ref.object == rw_repo.head.commit.parents[0] - # rename it + # Rename it. orig_obj = ref.object for name in ("refs/absname", "rela_name", "feature/rela_name"): ref_new_name = ref.rename(name) @@ -463,19 +462,19 @@ def test_head_reset(self, rw_repo): assert ref_new_name == ref # END for each name type - # References that don't exist trigger an error if we want to access them + # References that don't exist trigger an error if we want to access them. self.assertRaises(ValueError, getattr, Reference(rw_repo, "refs/doesntexist"), "commit") - # exists, fail unless we force + # Exists, fail unless we force. ex_ref_path = far_away_head.path self.assertRaises(OSError, ref.rename, ex_ref_path) - # if it points to the same commit it works + # If it points to the same commit it works. far_away_head.commit = ref.commit ref.rename(ex_ref_path) assert ref.path == ex_ref_path and ref.object == orig_obj assert ref.rename(ref.path).path == ex_ref_path # rename to same name - # create symbolic refs + # Create symbolic refs. symref_path = "symrefs/sym" symref = SymbolicReference.create(rw_repo, symref_path, cur_head.reference) assert symref.path == symref_path @@ -488,20 +487,20 @@ def test_head_reset(self, rw_repo): symref_path, cur_head.reference.commit, ) - # it works if the new ref points to the same reference + # It works if the new ref points to the same reference. assert SymbolicReference.create(rw_repo, symref.path, symref.reference).path == symref.path SymbolicReference.delete(rw_repo, symref) - # would raise if the symref wouldn't have been deletedpbl + # Would raise if the symref wouldn't have been deleted (probably). symref = SymbolicReference.create(rw_repo, symref_path, cur_head.reference) - # test symbolic references which are not at default locations like HEAD - # or FETCH_HEAD - they may also be at spots in refs of course + # Test symbolic references which are not at default locations like HEAD + # or FETCH_HEAD - they may also be at spots in refs of course. symbol_ref_path = "refs/symbol_ref" symref = SymbolicReference(rw_repo, symbol_ref_path) assert symref.path == symbol_ref_path symbol_ref_abspath = osp.join(rw_repo.git_dir, symref.path) - # set it + # Set it. symref.reference = new_head assert symref.reference == new_head assert osp.isfile(symbol_ref_abspath) @@ -516,10 +515,10 @@ def test_head_reset(self, rw_repo): assert not symref.is_detached # END for each ref - # create a new non-head ref just to be sure we handle it even if packed + # Create a new non-head ref just to be sure we handle it even if packed. Reference.create(rw_repo, full_ref) - # test ref listing - assure we have packed refs + # Test ref listing - make sure we have packed refs. rw_repo.git.pack_refs(all=True, prune=True) heads = rw_repo.heads assert heads @@ -527,15 +526,15 @@ def test_head_reset(self, rw_repo): assert active_branch in heads assert rw_repo.tags - # we should be able to iterate all symbolic refs as well - in that case - # we should expect only symbolic references to be returned + # We should be able to iterate all symbolic refs as well - in that case + # we should expect only symbolic references to be returned. for symref in SymbolicReference.iter_items(rw_repo): assert not symref.is_detached - # when iterating references, we can get references and symrefs - # when deleting all refs, I'd expect them to be gone ! Even from - # the packed ones - # For this to work, we must not be on any branch + # When iterating references, we can get references and symrefs + # when deleting all refs, I'd expect them to be gone! Even from + # the packed ones. + # For this to work, we must not be on any branch. rw_repo.head.reference = rw_repo.head.commit deleted_refs = set() for ref in Reference.iter_items(rw_repo): @@ -551,16 +550,15 @@ def test_head_reset(self, rw_repo): assert ref not in deleted_refs # END for each ref - # reattach head - head will not be returned if it is not a symbolic - # ref + # Reattach head - head will not be returned if it is not a symbolic ref. rw_repo.head.reference = Head.create(rw_repo, "master") - # At least the head should still exist + # At least the head should still exist. assert osp.isfile(osp.join(rw_repo.git_dir, "HEAD")) refs = list(SymbolicReference.iter_items(rw_repo)) assert len(refs) == 1 - # test creation of new refs from scratch + # Test creation of new refs from scratch. for path in ("basename", "dir/somename", "dir2/subdir/basename"): # REFERENCES ############ @@ -570,11 +568,11 @@ def test_head_reset(self, rw_repo): ref = Reference(rw_repo, fpath) assert ref == ref_fp - # can be created by assigning a commit + # Can be created by assigning a commit. ref.commit = rw_repo.head.commit assert ref.is_valid() - # if the assignment raises, the ref doesn't exist + # If the assignment raises, the ref doesn't exist. Reference.delete(ref.repo, ref.path) assert not ref.is_valid() self.assertRaises(ValueError, setattr, ref, "commit", "nonsense") @@ -614,7 +612,7 @@ def test_tag_message(self, rw_repo): assert tag_ref.tag.message == "test2" def test_dereference_recursive(self): - # for now, just test the HEAD + # For now, just test the HEAD. assert SymbolicReference.dereference_recursive(self.rorepo, "HEAD") def test_reflog(self): @@ -634,7 +632,7 @@ def test_refs_outside_repo(self): def test_validity_ref_names(self): check_ref = SymbolicReference._check_ref_name_valid - # Based on the rules specified in https://git-scm.com/docs/git-check-ref-format/#_description + # Based on the rules specified in https://git-scm.com/docs/git-check-ref-format/#_description. # Rule 1 self.assertRaises(ValueError, check_ref, ".ref/begins/with/dot") self.assertRaises(ValueError, check_ref, "ref/component/.begins/with/dot") @@ -665,5 +663,5 @@ def test_validity_ref_names(self): self.assertRaises(ValueError, check_ref, "@") # Rule 10 self.assertRaises(ValueError, check_ref, "ref/contain\\s/backslash") - # Valid reference name should not raise + # Valid reference name should not raise. check_ref("valid/ref/name") diff --git a/test/test_remote.py b/test/test_remote.py index 7144b2791..b102730ac 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -36,7 +36,7 @@ import os.path as osp -# assure we have repeatable results +# Make sure we have repeatable results. random.seed(0) @@ -50,8 +50,8 @@ def __init__(self): self._num_progress_messages = 0 def _parse_progress_line(self, line): - # we may remove the line later if it is dropped - # Keep it for debugging + # We may remove the line later if it is dropped. + # Keep it for debugging. self._seen_lines.append(line) rval = super(TestRemoteProgress, self)._parse_progress_line(line) return rval @@ -63,7 +63,7 @@ def line_dropped(self, line): pass def update(self, op_code, cur_count, max_count=None, message=""): - # check each stage only comes once + # Check each stage only comes once. op_id = op_code & self.OP_MASK assert op_id in (self.COUNTING, self.COMPRESSING, self.WRITING) @@ -85,15 +85,15 @@ def update(self, op_code, cur_count, max_count=None, message=""): self._num_progress_messages += 1 def make_assertion(self): - # we don't always receive messages + # We don't always receive messages. if not self._seen_lines: return - # sometimes objects are not compressed which is okay + # Sometimes objects are not compressed which is okay. assert len(self._seen_ops) in (2, 3), len(self._seen_ops) assert self._stages_per_op - # must have seen all stages + # Must have seen all stages. for _op, stages in self._stages_per_op.items(): assert stages & self.STAGE_MASK == self.STAGE_MASK # END for each op/stage @@ -151,7 +151,7 @@ def _do_test_push_result(self, results, remote): # END for each bitflag self.assertTrue(has_one) else: - # there must be a remote commit + # There must be a remote commit. if info.flags & info.DELETED == 0: self.assertIsInstance(info.local_ref, Reference) else: @@ -163,7 +163,7 @@ def _do_test_push_result(self, results, remote): if any(info.flags & info.ERROR for info in results): self.assertRaises(GitCommandError, results.raise_if_error) else: - # No errors, so this should do nothing + # No errors, so this should do nothing. results.raise_if_error() def _do_test_fetch_info(self, repo): @@ -177,8 +177,10 @@ def _do_test_fetch_info(self, repo): ) def _commit_random_file(self, repo): - # Create a file with a random name and random data and commit it to repo. - # Return the committed absolute file path + """Create a file with a random name and random data and commit it to a repo. + + :return: The committed absolute file path. + """ index = repo.index new_file = self._make_file(osp.basename(tempfile.mktemp()), str(random.random()), repo) index.add([new_file]) @@ -186,7 +188,7 @@ def _commit_random_file(self, repo): return new_file def _do_test_fetch(self, remote, rw_repo, remote_repo, **kwargs): - # specialized fetch testing to de-clutter the main test + """Specialized fetch testing to de-clutter the main test.""" self._do_test_fetch_info(rw_repo) def fetch_and_test(remote, **kwargs): @@ -202,16 +204,16 @@ def fetch_and_test(remote, **kwargs): def get_info(res, remote, name): return res["%s/%s" % (remote, name)] - # put remote head to master as it is guaranteed to exist + # Put remote head to master as it is guaranteed to exist. remote_repo.head.reference = remote_repo.heads.master res = fetch_and_test(remote, **kwargs) - # all up to date + # All up to date. for info in res: self.assertTrue(info.flags & info.HEAD_UPTODATE) - # rewind remote head to trigger rejection - # index must be false as remote is a bare repo + # Rewind remote head to trigger rejection. + # index must be false as remote is a bare repo. rhead = remote_repo.head remote_commit = rhead.commit rhead.reset("HEAD~2", index=False) @@ -221,50 +223,50 @@ def get_info(res, remote, name): self.assertTrue(master_info.flags & FetchInfo.FORCED_UPDATE) self.assertIsNotNone(master_info.note) - # normal fast forward - set head back to previous one + # Normal fast forward - set head back to previous one. rhead.commit = remote_commit res = fetch_and_test(remote) self.assertTrue(res[mkey].flags & FetchInfo.FAST_FORWARD) - # new remote branch + # New remote branch. new_remote_branch = Head.create(remote_repo, "new_branch") res = fetch_and_test(remote) new_branch_info = get_info(res, remote, new_remote_branch) self.assertTrue(new_branch_info.flags & FetchInfo.NEW_HEAD) - # remote branch rename ( causes creation of a new one locally ) + # Remote branch rename (causes creation of a new one locally). new_remote_branch.rename("other_branch_name") res = fetch_and_test(remote) other_branch_info = get_info(res, remote, new_remote_branch) self.assertEqual(other_branch_info.ref.commit, new_branch_info.ref.commit) - # remove new branch + # Remove new branch. Head.delete(new_remote_branch.repo, new_remote_branch) res = fetch_and_test(remote) - # deleted remote will not be fetched + # Deleted remote will not be fetched. self.assertRaises(IndexError, get_info, res, remote, new_remote_branch) - # prune stale tracking branches + # Prune stale tracking branches. stale_refs = remote.stale_refs self.assertEqual(len(stale_refs), 2) self.assertIsInstance(stale_refs[0], RemoteReference) RemoteReference.delete(rw_repo, *stale_refs) - # test single branch fetch with refspec including target remote + # Test single branch fetch with refspec including target remote. res = fetch_and_test(remote, refspec="master:refs/remotes/%s/master" % remote) self.assertEqual(len(res), 1) self.assertTrue(get_info(res, remote, "master")) - # ... with respec and no target + # ...with respec and no target. res = fetch_and_test(remote, refspec="master") self.assertEqual(len(res), 1) - # ... multiple refspecs ... works, but git command returns with error if one ref is wrong without - # doing anything. This is new in later binaries + # ...multiple refspecs...works, but git command returns with error if one ref is + # wrong without doing anything. This is new in later binaries. # res = fetch_and_test(remote, refspec=['master', 'fred']) # self.assertEqual(len(res), 1) - # add new tag reference + # Add new tag reference. rtag = TagReference.create(remote_repo, "1.0-RV_hello.there") res = fetch_and_test(remote, tags=True) tinfo = res[str(rtag)] @@ -272,10 +274,10 @@ def get_info(res, remote, name): self.assertEqual(tinfo.ref.commit, rtag.commit) self.assertTrue(tinfo.flags & tinfo.NEW_TAG) - # adjust the local tag commit + # Adjust the local tag commit. Reference.set_object(rtag, rhead.commit.parents[0].parents[0]) - # as of git 2.20 one cannot clobber local tags that have changed without + # As of git 2.20 one cannot clobber local tags that have changed without # specifying --force, and the test assumes you can clobber, so... force = None if rw_repo.git.version_info[:2] >= (2, 20): @@ -285,63 +287,63 @@ def get_info(res, remote, name): self.assertEqual(tinfo.commit, rtag.commit) self.assertTrue(tinfo.flags & tinfo.TAG_UPDATE) - # delete remote tag - local one will stay + # Delete remote tag - local one will stay. TagReference.delete(remote_repo, rtag) res = fetch_and_test(remote, tags=True) self.assertRaises(IndexError, get_info, res, remote, str(rtag)) - # provoke to receive actual objects to see what kind of output we have to - # expect. For that we need a remote transport protocol + # Provoke to receive actual objects to see what kind of output we have to + # expect. For that we need a remote transport protocol. # Create a new UN-shared repo and fetch into it after we pushed a change - # to the shared repo + # to the shared repo. other_repo_dir = tempfile.mktemp("other_repo") - # must clone with a local path for the repo implementation not to freak out - # as it wants local paths only ( which I can understand ) + # Must clone with a local path for the repo implementation not to freak out + # as it wants local paths only (which I can understand). other_repo = remote_repo.clone(other_repo_dir, shared=False) remote_repo_url = osp.basename(remote_repo.git_dir) # git-daemon runs with appropriate `--base-path`. remote_repo_url = Git.polish_url("git://localhost:%s/%s" % (GIT_DAEMON_PORT, remote_repo_url)) - # put origin to git-url + # Put origin to git-url. other_origin = other_repo.remotes.origin with other_origin.config_writer as cw: cw.set("url", remote_repo_url) - # it automatically creates alternates as remote_repo is shared as well. - # It will use the transport though and ignore alternates when fetching + # It automatically creates alternates as remote_repo is shared as well. + # It will use the transport though and ignore alternates when fetching. # assert not other_repo.alternates # this would fail - # assure we are in the right state + # Ensure we are in the right state. rw_repo.head.reset(remote.refs.master, working_tree=True) try: self._commit_random_file(rw_repo) remote.push(rw_repo.head.reference) - # here I would expect to see remote-information about packing + # Here I would expect to see remote-information about packing # objects and so on. Unfortunately, this does not happen # if we are redirecting the output - git explicitly checks for this - # and only provides progress information to ttys + # and only provides progress information to ttys. res = fetch_and_test(other_origin) finally: rmtree(other_repo_dir) # END test and cleanup def _assert_push_and_pull(self, remote, rw_repo, remote_repo): - # push our changes + # Push our changes. lhead = rw_repo.head - # assure we are on master and it is checked out where the remote is + # Ensure we are on master and it is checked out where the remote is. try: lhead.reference = rw_repo.heads.master except AttributeError: - # if the author is on a non-master branch, the clones might not have - # a local master yet. We simply create it + # If the author is on a non-master branch, the clones might not have + # a local master yet. We simply create it. lhead.reference = rw_repo.create_head("master") # END master handling lhead.reset(remote.refs.master, working_tree=True) - # push without spec should fail ( without further configuration ) + # Push without spec should fail (without further configuration) # well, works nicely # self.assertRaises(GitCommandError, remote.push) - # simple file push + # Simple file push. self._commit_random_file(rw_repo) progress = TestRemoteProgress() res = remote.push(lhead.reference, progress) @@ -349,23 +351,23 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): self._do_test_push_result(res, remote) progress.make_assertion() - # rejected - undo last commit + # Rejected - undo last commit. lhead.reset("HEAD~1") res = remote.push(lhead.reference) self.assertTrue(res[0].flags & PushInfo.ERROR) self.assertTrue(res[0].flags & PushInfo.REJECTED) self._do_test_push_result(res, remote) - # force rejected pull + # Force rejected pull. res = remote.push("+%s" % lhead.reference) self.assertEqual(res[0].flags & PushInfo.ERROR, 0) self.assertTrue(res[0].flags & PushInfo.FORCED_UPDATE) self._do_test_push_result(res, remote) - # invalid refspec + # Invalid refspec. self.assertRaises(GitCommandError, remote.push, "hellothere") - # push new tags + # Push new tags. progress = TestRemoteProgress() to_be_updated = "my_tag.1.0RV" new_tag = TagReference.create(rw_repo, to_be_updated) # @UnusedVariable @@ -375,28 +377,28 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): progress.make_assertion() self._do_test_push_result(res, remote) - # update push new tags - # Rejection is default + # Update push new tags. + # Rejection is default. new_tag = TagReference.create(rw_repo, to_be_updated, reference="HEAD~1", force=True) res = remote.push(tags=True) self._do_test_push_result(res, remote) self.assertTrue(res[-1].flags & PushInfo.REJECTED) self.assertTrue(res[-1].flags & PushInfo.ERROR) - # push force this tag + # Force push this tag. res = remote.push("+%s" % new_tag.path) self.assertEqual(res[-1].flags & PushInfo.ERROR, 0) self.assertTrue(res[-1].flags & PushInfo.FORCED_UPDATE) - # delete tag - have to do it using refspec + # Delete tag - have to do it using refspec. res = remote.push(":%s" % new_tag.path) self._do_test_push_result(res, remote) self.assertTrue(res[0].flags & PushInfo.DELETED) # Currently progress is not properly transferred, especially not using - # the git daemon + # the git daemon. # progress.assert_received_message() - # push new branch + # Push new branch. new_head = Head.create(rw_repo, "my_new_branch") progress = TestRemoteProgress() res = remote.push(new_head, progress) @@ -405,7 +407,7 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): progress.make_assertion() self._do_test_push_result(res, remote) - # rejected stale delete + # Rejected stale delete. force_with_lease = "%s:0000000000000000000000000000000000000000" % new_head.path res = remote.push(":%s" % new_head.path, force_with_lease=force_with_lease) self.assertTrue(res[0].flags & PushInfo.ERROR) @@ -413,7 +415,7 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): self.assertIsNone(res[0].local_ref) self._do_test_push_result(res, remote) - # delete new branch on the remote end and locally + # Delete new branch on the remote end and locally. res = remote.push(":%s" % new_head.path) self._do_test_push_result(res, remote) Head.delete(rw_repo, new_head) @@ -425,8 +427,8 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): remote.pull("master", kill_after_timeout=10.0) - # cleanup - delete created tags and branches as we are in an innerloop on - # the same repository + # Cleanup - delete created tags and branches as we are in an inner loop on + # the same repository. TagReference.delete(rw_repo, new_tag, other_tag) remote.push(":%s" % other_tag.path, kill_after_timeout=10.0) @@ -442,7 +444,7 @@ def test_base(self, rw_repo, remote_repo): self.assertEqual(remote, remote) self.assertNotEqual(str(remote), repr(remote)) remote_set.add(remote) - remote_set.add(remote) # should already exist + remote_set.add(remote) # Should already exist. # REFS refs = remote.refs self.assertTrue(refs) @@ -452,17 +454,17 @@ def test_base(self, rw_repo, remote_repo): # END for each ref # OPTIONS - # cannot use 'fetch' key anymore as it is now a method + # Cannot use 'fetch' key anymore as it is now a method. for opt in ("url",): val = getattr(remote, opt) reader = remote.config_reader assert reader.get(opt) == val assert reader.get_value(opt, None) == val - # unable to write with a reader + # Unable to write with a reader. self.assertRaises(IOError, reader.set, opt, "test") - # change value + # Change value. with remote.config_writer as writer: new_val = "myval" writer.set(opt, new_val) @@ -477,7 +479,7 @@ def test_base(self, rw_repo, remote_repo): prev_name = remote.name self.assertEqual(remote.rename(other_name), remote) self.assertNotEqual(prev_name, remote.name) - # multiple times + # Multiple times. for _ in range(2): self.assertEqual(remote.rename(prev_name).name, prev_name) # END for each rename ( back to prev_name ) @@ -487,7 +489,7 @@ def test_base(self, rw_repo, remote_repo): # FETCH TESTING # Only for remotes - local cases are the same or less complicated - # as additional progress information will never be emitted + # as additional progress information will never be emitted. if remote.name == "daemon_origin": self._do_test_fetch(remote, rw_repo, remote_repo, kill_after_timeout=10.0) ran_fetch_test = True @@ -503,10 +505,10 @@ def test_base(self, rw_repo, remote_repo): origin = rw_repo.remote("origin") assert origin == rw_repo.remotes.origin - # Verify we can handle prunes when fetching + # Verify we can handle prunes when fetching. # stderr lines look like this: x [deleted] (none) -> origin/experiment-2012 - # These should just be skipped - # If we don't have a manual checkout, we can't actually assume there are any non-master branches + # These should just be skipped. + # If we don't have a manual checkout, we can't actually assume there are any non-master branches. remote_repo.create_head("myone_for_deletion") # Get the branch - to be pruned later origin.fetch() @@ -534,13 +536,13 @@ def test_creation_and_removal(self, bare_rw_repo): self.assertIn(remote, bare_rw_repo.remotes) self.assertTrue(remote.exists()) - # create same one again + # Create same one again. self.assertRaises(GitCommandError, Remote.create, bare_rw_repo, *arg_list) Remote.remove(bare_rw_repo, new_name) - self.assertTrue(remote.exists()) # We still have a cache that doesn't know we were deleted by name + self.assertTrue(remote.exists()) # We still have a cache that doesn't know we were deleted by name. remote._clear_cache() - assert not remote.exists() # Cache should be renewed now. This is an issue ... + assert not remote.exists() # Cache should be renewed now. This is an issue... for remote in bare_rw_repo.remotes: if remote.name == new_name: @@ -548,11 +550,11 @@ def test_creation_and_removal(self, bare_rw_repo): # END if deleted remote matches existing remote's name # END for each remote - # Issue #262 - the next call would fail if bug wasn't fixed + # Issue #262 - the next call would fail if bug wasn't fixed. bare_rw_repo.create_remote("bogus", "/bogus/path", mirror="push") def test_fetch_info(self): - # assure we can handle remote-tracking branches + # Ensure we can handle remote-tracking branches. fetch_info_line_fmt = "c437ee5deb8d00cf02f03720693e4c802e99f390 not-for-merge %s '0.3' of " fetch_info_line_fmt += "git://github.com/gitpython-developers/GitPython" remote_info_line_fmt = "* [new branch] nomatter -> %s" @@ -573,8 +575,8 @@ def test_fetch_info(self): assert not fi.ref.is_valid() self.assertEqual(fi.ref.name, "local/master") - # handles non-default refspecs: One can specify a different path in refs/remotes - # or a special path just in refs/something for instance + # Handles non-default refspecs: One can specify a different path in refs/remotes + # or a special path just in refs/something for instance. fi = FetchInfo._from_line( self.rorepo, @@ -585,7 +587,7 @@ def test_fetch_info(self): self.assertIsInstance(fi.ref, TagReference) assert fi.ref.path.startswith("refs/tags"), fi.ref.path - # it could be in a remote direcftory though + # It could be in a remote directory though. fi = FetchInfo._from_line( self.rorepo, remote_info_line_fmt % "remotename/tags/tagname", @@ -595,14 +597,14 @@ def test_fetch_info(self): self.assertIsInstance(fi.ref, TagReference) assert fi.ref.path.startswith("refs/remotes/"), fi.ref.path - # it can also be anywhere ! + # It can also be anywhere! tag_path = "refs/something/remotename/tags/tagname" fi = FetchInfo._from_line(self.rorepo, remote_info_line_fmt % tag_path, fetch_info_line_fmt % "tag") self.assertIsInstance(fi.ref, TagReference) self.assertEqual(fi.ref.path, tag_path) - # branches default to refs/remotes + # Branches default to refs/remotes. fi = FetchInfo._from_line( self.rorepo, remote_info_line_fmt % "remotename/branch", @@ -612,7 +614,7 @@ def test_fetch_info(self): self.assertIsInstance(fi.ref, RemoteReference) self.assertEqual(fi.ref.remote_name, "remotename") - # but you can force it anywhere, in which case we only have a references + # But you can force it anywhere, in which case we only have a references. fi = FetchInfo._from_line( self.rorepo, remote_info_line_fmt % "refs/something/branch", @@ -639,46 +641,46 @@ def test_uncommon_branch_names(self): @with_rw_repo("HEAD", bare=False) def test_multiple_urls(self, rw_repo): - # test addresses + # Test addresses. test1 = "https://github.com/gitpython-developers/GitPython" test2 = "https://github.com/gitpython-developers/gitdb" test3 = "https://github.com/gitpython-developers/smmap" remote = rw_repo.remotes[0] - # Testing setting a single URL + # Test setting a single URL. remote.set_url(test1) self.assertEqual(list(remote.urls), [test1]) - # Testing replacing that single URL + # Test replacing that single URL. remote.set_url(test1) self.assertEqual(list(remote.urls), [test1]) - # Testing adding new URLs + # Test adding new URLs. remote.set_url(test2, add=True) self.assertEqual(list(remote.urls), [test1, test2]) remote.set_url(test3, add=True) self.assertEqual(list(remote.urls), [test1, test2, test3]) - # Testing removing an URL + # Test removing a URL. remote.set_url(test2, delete=True) self.assertEqual(list(remote.urls), [test1, test3]) - # Testing changing an URL + # Test changing a URL. remote.set_url(test2, test3) self.assertEqual(list(remote.urls), [test1, test2]) # will raise: fatal: --add --delete doesn't make sense self.assertRaises(GitCommandError, remote.set_url, test2, add=True, delete=True) - # Testing on another remote, with the add/delete URL + # Test on another remote, with the add/delete URL. remote = rw_repo.create_remote("another", url=test1) remote.add_url(test2) self.assertEqual(list(remote.urls), [test1, test2]) remote.add_url(test3) self.assertEqual(list(remote.urls), [test1, test2, test3]) - # Testing removing all the URLs + # Test removing all the URLs. remote.delete_url(test2) self.assertEqual(list(remote.urls), [test1, test3]) remote.delete_url(test1) self.assertEqual(list(remote.urls), [test3]) - # will raise fatal: Will not delete all non-push URLs + # Will raise fatal: Will not delete all non-push URLs. self.assertRaises(GitCommandError, remote.delete_url, test3) def test_fetch_error(self): @@ -972,14 +974,13 @@ def test_push_unsafe_options_allowed(self, rw_repo): class TestTimeouts(TestBase): @with_rw_repo("HEAD", bare=False) def test_timeout_funcs(self, repo): - # Force error code to prevent a race condition if the python thread is - # slow + # Force error code to prevent a race condition if the python thread is slow. default = Git.AutoInterrupt._status_code_if_terminate Git.AutoInterrupt._status_code_if_terminate = -15 - for function in ["pull", "fetch"]: # can't get push to timeout + for function in ["pull", "fetch"]: # Can't get push to time out. f = getattr(repo.remotes.origin, function) - assert f is not None # Make sure these functions exist - _ = f() # Make sure the function runs + assert f is not None # Make sure these functions exist. + _ = f() # Make sure the function runs. with pytest.raises(GitCommandError, match="kill_after_timeout=0 s"): f(kill_after_timeout=0) diff --git a/test/test_repo.py b/test/test_repo.py index 364b895fb..c7c13dc7f 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -4,6 +4,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import glob import io from io import BytesIO @@ -163,28 +164,29 @@ def test_trees(self): self.assertEqual(num_trees, mc) def _assert_empty_repo(self, repo): - # test all kinds of things with an empty, freshly initialized repo. - # It should throw good errors + """Test all kinds of things with an empty, freshly initialized repo. - # entries should be empty + It should throw good errors. + """ + # Entries should be empty. self.assertEqual(len(repo.index.entries), 0) - # head is accessible + # head is accessible. assert repo.head assert repo.head.ref assert not repo.head.is_valid() - # we can change the head to some other ref + # We can change the head to some other ref. head_ref = Head.from_path(repo, Head.to_full_path("some_head")) assert not head_ref.is_valid() repo.head.ref = head_ref - # is_dirty can handle all kwargs + # is_dirty can handle all kwargs. for args in ((1, 0, 0), (0, 1, 0), (0, 0, 1)): assert not repo.is_dirty(*args) # END for each arg - # we can add a file to the index ( if we are not bare ) + # We can add a file to the index (if we are not bare). if not repo.bare: pass # END test repos with working tree @@ -201,7 +203,7 @@ def test_clone_from_keeps_env(self, rw_dir): @with_rw_directory def test_date_format(self, rw_dir): repo = Repo.init(osp.join(rw_dir, "repo")) - # @-timestamp is the format used by git commit hooks + # @-timestamp is the format used by git commit hooks. repo.index.commit("Commit messages", commit_date="@1400000000 +0000") @with_rw_directory @@ -262,7 +264,7 @@ def test_leaking_password_in_clone_logs(self, rw_dir): ) except GitCommandError as err: assert password not in str(err), "The error message '%s' should not contain the password" % err - # Working example from a blank private project + # Working example from a blank private project. Repo.clone_from( url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", to_path=rw_dir, @@ -514,12 +516,12 @@ def test_init(self): try: rmtree(clone_path) except OSError: - # when relative paths are used, the clone may actually be inside - # of the parent directory + # When relative paths are used, the clone may actually be inside + # of the parent directory. pass # END exception handling - # try again, this time with the absolute version + # Try again, this time with the absolute version. rc = Repo.clone_from(r.git_dir, clone_path) self._assert_empty_repo(rc) @@ -527,8 +529,8 @@ def test_init(self): try: rmtree(clone_path) except OSError: - # when relative paths are used, the clone may actually be inside - # of the parent directory + # When relative paths are used, the clone may actually be inside + # of the parent directory. pass # END exception handling @@ -675,11 +677,11 @@ def test_should_display_blame_information(self, git): lambda: c.message, ) - # test the 'lines per commit' entries + # Test the 'lines per commit' entries. tlist = b[0][1] self.assertTrue(tlist) self.assertTrue(isinstance(tlist[0], str)) - self.assertTrue(len(tlist) < sum(len(t) for t in tlist)) # test for single-char bug + self.assertTrue(len(tlist) < sum(len(t) for t in tlist)) # Test for single-char bug. # BINARY BLAME git.return_value = fixture("blame_binary") @@ -688,7 +690,7 @@ def test_should_display_blame_information(self, git): def test_blame_real(self): c = 0 - nml = 0 # amount of multi-lines per blame + nml = 0 # Amount of multi-lines per blame. for item in self.rorepo.head.commit.tree.traverse( predicate=lambda i, d: i.type == "blob" and i.path.endswith(".py") ): @@ -702,14 +704,14 @@ def test_blame_real(self): @mock.patch.object(Git, "_call_process") def test_blame_incremental(self, git): - # loop over two fixtures, create a test fixture for 2.11.1+ syntax + # Loop over two fixtures, create a test fixture for 2.11.1+ syntax. for git_fixture in ("blame_incremental", "blame_incremental_2.11.1_plus"): git.return_value = fixture(git_fixture) blame_output = self.rorepo.blame_incremental("9debf6b0aafb6f7781ea9d1383c86939a1aacde3", "AUTHORS") blame_output = list(blame_output) self.assertEqual(len(blame_output), 5) - # Check all outputted line numbers + # Check all outputted line numbers. ranges = flatten([entry.linenos for entry in blame_output]) self.assertEqual( ranges, @@ -727,13 +729,13 @@ def test_blame_incremental(self, git): commits = [entry.commit.hexsha[:7] for entry in blame_output] self.assertEqual(commits, ["82b8902", "82b8902", "c76852d", "c76852d", "c76852d"]) - # Original filenames + # Original filenames. self.assertSequenceEqual( [entry.orig_path for entry in blame_output], ["AUTHORS"] * len(blame_output), ) - # Original line numbers + # Original line numbers. orig_ranges = flatten([entry.orig_linenos for entry in blame_output]) self.assertEqual( orig_ranges, @@ -780,7 +782,7 @@ def test_untracked_files(self, rwrepo): untracked_files = rwrepo.untracked_files num_recently_untracked = len(untracked_files) - # assure we have all names - they are relative to the git-dir + # Ensure we have all names - they are relative to the git-dir. num_test_untracked = 0 for utfile in untracked_files: num_test_untracked += join_path_native(base, utfile) in files @@ -791,9 +793,9 @@ def test_untracked_files(self, rwrepo): # end for each run def test_config_reader(self): - reader = self.rorepo.config_reader() # all config files + reader = self.rorepo.config_reader() # All config files. assert reader.read_only - reader = self.rorepo.config_reader("repository") # single config file + reader = self.rorepo.config_reader("repository") # Single config file. assert reader.read_only def test_config_writer(self): @@ -802,8 +804,8 @@ def test_config_writer(self): with self.rorepo.config_writer(config_level) as writer: self.assertFalse(writer.read_only) except IOError: - # its okay not to get a writer for some configuration files if we - # have no permissions + # It's okay not to get a writer for some configuration files if we + # have no permissions. pass def test_config_level_paths(self): @@ -811,8 +813,8 @@ def test_config_level_paths(self): assert self.rorepo._get_config_path(config_level) def test_creation_deletion(self): - # just a very quick test to assure it generally works. There are - # specialized cases in the test_refs module + # Just a very quick test to assure it generally works. There are + # specialized cases in the test_refs module. head = self.rorepo.create_head("new_head", "HEAD~1") self.rorepo.delete_head(head) @@ -828,7 +830,7 @@ def test_creation_deletion(self): self.rorepo.delete_remote(remote) def test_comparison_and_hash(self): - # this is only a preliminary test, more testing done in test_index + # This is only a preliminary test, more testing done in test_index. self.assertEqual(self.rorepo, self.rorepo) self.assertFalse(self.rorepo != self.rorepo) self.assertEqual(len({self.rorepo, self.rorepo}), 1) @@ -844,8 +846,8 @@ def test_tilde_and_env_vars_in_repo_path(self, rw_dir): Repo.init(osp.join("$FOO", "test.git"), bare=True) def test_git_cmd(self): - # test CatFileContentStream, just to be very sure we have no fencepost errors - # last \n is the terminating newline that it expects + # Test CatFileContentStream, just to be very sure we have no fencepost errors. + # The last \n is the terminating newline that it expects. l1 = b"0123456789\n" l2 = b"abcdefghijklmnopqrstxy\n" l3 = b"z\n" @@ -853,8 +855,8 @@ def test_git_cmd(self): l1p = l1[:5] - # full size - # size is without terminating newline + # Full size. + # Size is without terminating newline. def mkfull(): return Git.CatFileContentStream(len(d) - 1, BytesIO(d)) @@ -868,7 +870,7 @@ def mktiny(): lines = s.readlines() self.assertEqual(len(lines), 3) self.assertTrue(lines[-1].endswith(b"\n"), lines[-1]) - self.assertEqual(s._stream.tell(), len(d)) # must have scrubbed to the end + self.assertEqual(s._stream.tell(), len(d)) # Must have scrubbed to the end. # realines line limit s = mkfull() @@ -911,7 +913,7 @@ def mktiny(): s = mkfull() self.assertEqual(s.read(5), l1p) self.assertEqual(s.read(6), l1[5:]) - self.assertEqual(s._stream.tell(), 5 + 6) # its not yet done + self.assertEqual(s._stream.tell(), 5 + 6) # It's not yet done. # read tiny s = mktiny() @@ -926,7 +928,7 @@ def _assert_rev_parse_types(self, name, rev_obj): if rev_obj.type == "tag": rev_obj = rev_obj.object - # tree and blob type + # Tree and blob type. obj = rev_parse(name + "^{tree}") self.assertEqual(obj, rev_obj.tree) @@ -946,13 +948,13 @@ def _assert_rev_parse(self, name): obj = orig_obj # END deref tags by default - # try history + # Try history rev = name + "~" obj2 = rev_parse(rev) self.assertEqual(obj2, obj.parents[0]) self._assert_rev_parse_types(rev, obj2) - # history with number + # History with number ni = 11 history = [obj.parents[0]] for _ in range(ni): @@ -966,13 +968,13 @@ def _assert_rev_parse(self, name): self._assert_rev_parse_types(rev, obj2) # END history check - # parent ( default ) + # Parent (default) rev = name + "^" obj2 = rev_parse(rev) self.assertEqual(obj2, obj.parents[0]) self._assert_rev_parse_types(rev, obj2) - # parent with number + # Parent with number for pn, parent in enumerate(obj.parents): rev = name + "^%i" % (pn + 1) self.assertEqual(rev_parse(rev), parent) @@ -983,17 +985,17 @@ def _assert_rev_parse(self, name): @with_rw_repo("HEAD", bare=False) def test_rw_rev_parse(self, rwrepo): - # verify it does not confuse branches with hexsha ids + # Verify it does not confuse branches with hexsha ids. ahead = rwrepo.create_head("aaaaaaaa") assert rwrepo.rev_parse(str(ahead)) == ahead.commit def test_rev_parse(self): rev_parse = self.rorepo.rev_parse - # try special case: This one failed at some point, make sure its fixed + # Try special case: This one failed at some point, make sure its fixed. self.assertEqual(rev_parse("33ebe").hexsha, "33ebe7acec14b25c5f84f35a664803fcab2f7781") - # start from reference + # Start from reference. num_resolved = 0 for ref_no, ref in enumerate(Reference.iter_items(self.rorepo)): @@ -1006,7 +1008,7 @@ def test_rev_parse(self): num_resolved += 1 except (BadName, BadObject): print("failed on %s" % path_section) - # is fine, in case we have something like 112, which belongs to remotes/rname/merge-requests/112 + # This is fine if we have something like 112, which belongs to remotes/rname/merge-requests/112. # END exception handling # END for each token if ref_no == 3 - 1: @@ -1014,21 +1016,21 @@ def test_rev_parse(self): # END for each reference assert num_resolved - # it works with tags ! + # It works with tags! tag = self._assert_rev_parse("0.1.4") self.assertEqual(tag.type, "tag") - # try full sha directly ( including type conversion ) + # try full sha directly (including type conversion). self.assertEqual(tag.object, rev_parse(tag.object.hexsha)) self._assert_rev_parse_types(tag.object.hexsha, tag.object) - # multiple tree types result in the same tree: HEAD^{tree}^{tree}:CHANGES + # Multiple tree types result in the same tree: HEAD^{tree}^{tree}:CHANGES rev = "0.1.4^{tree}^{tree}" self.assertEqual(rev_parse(rev), tag.object.tree) self.assertEqual(rev_parse(rev + ":CHANGES"), tag.object.tree["CHANGES"]) - # try to get parents from first revision - it should fail as no such revision - # exists + # Try to get parents from first revision - it should fail as no such revision + # exists. first_rev = "33ebe7acec14b25c5f84f35a664803fcab2f7781" commit = rev_parse(first_rev) self.assertEqual(len(commit.parents), 0) @@ -1036,14 +1038,14 @@ def test_rev_parse(self): self.assertRaises(BadName, rev_parse, first_rev + "~") self.assertRaises(BadName, rev_parse, first_rev + "^") - # short SHA1 + # Short SHA1. commit2 = rev_parse(first_rev[:20]) self.assertEqual(commit2, commit) commit2 = rev_parse(first_rev[:5]) self.assertEqual(commit2, commit) - # todo: dereference tag into a blob 0.1.7^{blob} - quite a special one - # needs a tag which points to a blob + # TODO: Dereference tag into a blob 0.1.7^{blob} - quite a special one. + # Needs a tag which points to a blob. # ref^0 returns commit being pointed to, same with ref~0, and ^{} tag = rev_parse("0.1.4") @@ -1051,7 +1053,7 @@ def test_rev_parse(self): self.assertEqual(tag.object, rev_parse("0.1.4%s" % token)) # END handle multiple tokens - # try partial parsing + # Try partial parsing. max_items = 40 for i, binsha in enumerate(self.rorepo.odb.sha_iter()): self.assertEqual( @@ -1059,41 +1061,41 @@ def test_rev_parse(self): binsha, ) if i > max_items: - # this is rather slow currently, as rev_parse returns an object - # which requires accessing packs, it has some additional overhead + # This is rather slow currently, as rev_parse returns an object that + # requires accessing packs, so it has some additional overhead. break # END for each binsha in repo - # missing closing brace commit^{tree + # Missing closing brace: commit^{tree self.assertRaises(ValueError, rev_parse, "0.1.4^{tree") - # missing starting brace + # Missing starting brace. self.assertRaises(ValueError, rev_parse, "0.1.4^tree}") # REVLOG ####### head = self.rorepo.head - # need to specify a ref when using the @ syntax + # Need to specify a ref when using the @ syntax. self.assertRaises(BadObject, rev_parse, "%s@{0}" % head.commit.hexsha) - # uses HEAD.ref by default + # Uses HEAD.ref by default. self.assertEqual(rev_parse("@{0}"), head.commit) if not head.is_detached: refspec = "%s@{0}" % head.ref.name self.assertEqual(rev_parse(refspec), head.ref.commit) - # all additional specs work as well + # All additional specs work as well. self.assertEqual(rev_parse(refspec + "^{tree}"), head.commit.tree) self.assertEqual(rev_parse(refspec + ":CHANGES").type, "blob") # END operate on non-detached head - # position doesn't exist + # Position doesn't exist. self.assertRaises(IndexError, rev_parse, "@{10000}") - # currently, nothing more is supported + # Currently, nothing more is supported. self.assertRaises(NotImplementedError, rev_parse, "@{1 week ago}") - # the last position + # The last position. assert rev_parse("@{1}") != head.commit def test_repo_odbtype(self): @@ -1114,12 +1116,12 @@ def test_submodules(self): @with_rw_repo("HEAD", bare=False) def test_submodule_update(self, rwrepo): - # fails in bare mode + # Fails in bare mode. rwrepo._bare = True self.assertRaises(InvalidGitRepositoryError, rwrepo.submodule_update) rwrepo._bare = False - # test create submodule + # Test submodule creation. sm = rwrepo.submodules[0] sm = rwrepo.create_submodule( "my_new_sub", @@ -1128,7 +1130,7 @@ def test_submodule_update(self, rwrepo): ) self.assertIsInstance(sm, Submodule) - # note: the rest of this functionality is tested in test_submodule + # NOTE: the rest of this functionality is tested in test_submodule. @with_rw_repo("HEAD") def test_git_file(self, rwrepo): @@ -1154,11 +1156,11 @@ def last_commit(repo, rev, path): commit = next(repo.iter_commits(rev, path, max_count=1)) commit.tree[path] - # This is based on this comment + # This is based on this comment: # https://github.com/gitpython-developers/GitPython/issues/60#issuecomment-23558741 - # And we expect to set max handles to a low value, like 64 + # And we expect to set max handles to a low value, like 64. # You should set ulimit -n X, see .travis.yml - # The loops below would easily create 500 handles if these would leak (4 pipes + multiple mapped files) + # The loops below would easily create 500 handles if these would leak (4 pipes + multiple mapped files). for _ in range(64): for repo_type in (GitCmdObjectDB, GitDB): repo = Repo(self.rorepo.working_tree_dir, odbt=repo_type) @@ -1174,13 +1176,13 @@ def test_remote_method(self): def test_empty_repo(self, rw_dir): """Assure we can handle empty repositories""" r = Repo.init(rw_dir, mkdir=False) - # It's ok not to be able to iterate a commit, as there is none + # It's ok not to be able to iterate a commit, as there is none. self.assertRaises(ValueError, r.iter_commits) self.assertEqual(r.active_branch.name, "master") assert not r.active_branch.is_valid(), "Branch is yet to be born" - # actually, when trying to create a new branch without a commit, git itself fails - # We should, however, not fail ungracefully + # Actually, when trying to create a new branch without a commit, git itself fails. + # We should, however, not fail ungracefully. self.assertRaises(BadName, r.create_head, "foo") self.assertRaises(BadName, r.create_head, "master") # It's expected to not be able to access a tree @@ -1191,7 +1193,7 @@ def test_empty_repo(self, rw_dir): r.index.add([new_file_path]) r.index.commit("initial commit\nBAD MESSAGE 1\n") - # Now a branch should be creatable + # Now a branch should be creatable. nb = r.create_head("foo") assert nb.is_valid() @@ -1214,7 +1216,7 @@ def test_merge_base(self): self.assertRaises(ValueError, repo.merge_base) self.assertRaises(ValueError, repo.merge_base, "foo") - # two commit merge-base + # Two commit merge-base. res = repo.merge_base(c1, c2) self.assertIsInstance(res, list) self.assertEqual(len(res), 1) @@ -1227,7 +1229,7 @@ def test_merge_base(self): self.assertEqual(len(res), 1) # end for each keyword signalling all merge-bases to be returned - # Test for no merge base - can't do as we have + # Test for no merge base - can't do as we have. self.assertRaises(GitCommandError, repo.merge_base, c1, "ffffff") def test_is_ancestor(self): @@ -1254,22 +1256,22 @@ def test_is_valid_object(self): tree_sha = "960b40fe36" tag_sha = "42c2f60c43" - # Check for valid objects + # Check for valid objects. self.assertTrue(repo.is_valid_object(commit_sha)) self.assertTrue(repo.is_valid_object(blob_sha)) self.assertTrue(repo.is_valid_object(tree_sha)) self.assertTrue(repo.is_valid_object(tag_sha)) - # Check for valid objects of specific type + # Check for valid objects of specific type. self.assertTrue(repo.is_valid_object(commit_sha, "commit")) self.assertTrue(repo.is_valid_object(blob_sha, "blob")) self.assertTrue(repo.is_valid_object(tree_sha, "tree")) self.assertTrue(repo.is_valid_object(tag_sha, "tag")) - # Check for invalid objects + # Check for invalid objects. self.assertFalse(repo.is_valid_object(b"1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a", "blob")) - # Check for invalid objects of specific type + # Check for invalid objects of specific type. self.assertFalse(repo.is_valid_object(commit_sha, "blob")) self.assertFalse(repo.is_valid_object(blob_sha, "commit")) self.assertFalse(repo.is_valid_object(tree_sha, "commit")) @@ -1290,16 +1292,16 @@ def test_git_work_tree_dotgit(self, rw_dir): worktree_path = cygpath(worktree_path) rw_master.git.worktree("add", worktree_path, branch.name) - # this ensures that we can read the repo's gitdir correctly + # This ensures that we can read the repo's gitdir correctly. repo = Repo(worktree_path) self.assertIsInstance(repo, Repo) - # this ensures we're able to actually read the refs in the tree, which + # This ensures we're able to actually read the refs in the tree, which # means we can read commondir correctly. commit = repo.head.commit self.assertIsInstance(commit, Object) - # this ensures we can read the remotes, which confirms we're reading + # This ensures we can read the remotes, which confirms we're reading # the config correctly. origin = repo.remotes.origin self.assertIsInstance(origin, Remote) @@ -1308,11 +1310,11 @@ def test_git_work_tree_dotgit(self, rw_dir): @with_rw_directory def test_git_work_tree_env(self, rw_dir): - """Check that we yield to GIT_WORK_TREE""" - # clone a repo - # move .git directory to a subdirectory - # set GIT_DIR and GIT_WORK_TREE appropriately - # check that repo.working_tree_dir == rw_dir + """Check that we yield to GIT_WORK_TREE.""" + # Clone a repo. + # Move .git directory to a subdirectory. + # Set GIT_DIR and GIT_WORK_TREE appropriately. + # Check that: repo.working_tree_dir == rw_dir self.rorepo.clone(join_path_native(rw_dir, "master_repo")) @@ -1376,7 +1378,7 @@ def test_clone_command_injection(self, rw_repo): rw_repo.clone(payload) assert not unexpected_file.exists() - # A repo was cloned with the payload as name + # A repo was cloned with the payload as name. assert pathlib.Path(payload).exists() @with_rw_repo("HEAD") diff --git a/test/test_submodule.py b/test/test_submodule.py index 31a555ce2..e6e54c858 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import contextlib import os import shutil @@ -74,48 +75,48 @@ def tearDown(self): def _do_base_tests(self, rwrepo): """Perform all tests in the given repository, it may be bare or nonbare""" - # manual instantiation + # Manual instantiation. smm = Submodule(rwrepo, "\0" * 20) - # name needs to be set in advance + # Name needs to be set in advance. self.assertRaises(AttributeError, getattr, smm, "name") - # iterate - 1 submodule + # Iterate - 1 submodule. sms = Submodule.list_items(rwrepo, self.k_subm_current) assert len(sms) == 1 sm = sms[0] - # at a different time, there is None + # At a different time, there is None. assert len(Submodule.list_items(rwrepo, self.k_no_subm_tag)) == 0 assert sm.path == "git/ext/gitdb" - assert sm.path != sm.name # in our case, we have ids there, which don't equal the path + assert sm.path != sm.name # In our case, we have ids there, which don't equal the path. assert sm.url.endswith("github.com/gitpython-developers/gitdb.git") assert sm.branch_path == "refs/heads/master" # the default ... assert sm.branch_name == "master" assert sm.parent_commit == rwrepo.head.commit - # size is always 0 + # Size is always 0. assert sm.size == 0 - # the module is not checked-out yet + # The module is not checked-out yet. self.assertRaises(InvalidGitRepositoryError, sm.module) - # which is why we can't get the branch either - it points into the module() repository + # ...which is why we can't get the branch either - it points into the module() repository. self.assertRaises(InvalidGitRepositoryError, getattr, sm, "branch") - # branch_path works, as its just a string + # branch_path works, as it's just a string. assert isinstance(sm.branch_path, str) - # some commits earlier we still have a submodule, but its at a different commit + # Some commits earlier we still have a submodule, but it's at a different commit. smold = next(Submodule.iter_items(rwrepo, self.k_subm_changed)) assert smold.binsha != sm.binsha assert smold != sm # the name changed - # force it to reread its information + # Force it to reread its information. del smold._url smold.url == sm.url # noqa: B015 # FIXME: Should this be an assertion? - # test config_reader/writer methods + # Test config_reader/writer methods. sm.config_reader() - new_smclone_path = None # keep custom paths for later + new_smclone_path = None # Keep custom paths for later. new_csmclone_path = None # if rwrepo.bare: with self.assertRaises(InvalidGitRepositoryError): @@ -123,7 +124,7 @@ def _do_base_tests(self, rwrepo): pass else: with sm.config_writer() as writer: - # for faster checkout, set the url to the local path + # For faster checkout, set the url to the local path. new_smclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path)) writer.set_value("url", new_smclone_path) writer.release() @@ -132,76 +133,76 @@ def _do_base_tests(self, rwrepo): # END handle bare repo smold.config_reader() - # cannot get a writer on historical submodules + # Cannot get a writer on historical submodules. if not rwrepo.bare: with self.assertRaises(ValueError): with smold.config_writer(): pass # END handle bare repo - # make the old into a new - this doesn't work as the name changed + # Make the old into a new - this doesn't work as the name changed. self.assertRaises(ValueError, smold.set_parent_commit, self.k_subm_current) # the sha is properly updated smold.set_parent_commit(self.k_subm_changed + "~1") assert smold.binsha != sm.binsha - # raises if the sm didn't exist in new parent - it keeps its - # parent_commit unchanged + # Raises if the sm didn't exist in new parent - it keeps its + # parent_commit unchanged. self.assertRaises(ValueError, smold.set_parent_commit, self.k_no_subm_tag) - # TEST TODO: if a path in the gitmodules file, but not in the index, it raises + # TEST TODO: If a path is in the .gitmodules file, but not in the index, it raises. # TEST UPDATE ############## - # module retrieval is not always possible + # Module retrieval is not always possible. if rwrepo.bare: self.assertRaises(InvalidGitRepositoryError, sm.module) self.assertRaises(InvalidGitRepositoryError, sm.remove) self.assertRaises(InvalidGitRepositoryError, sm.add, rwrepo, "here", "there") else: - # its not checked out in our case + # It's not checked out in our case. self.assertRaises(InvalidGitRepositoryError, sm.module) assert not sm.module_exists() - # currently there is only one submodule + # Currently there is only one submodule. assert len(list(rwrepo.iter_submodules())) == 1 assert sm.binsha != "\0" * 20 # TEST ADD ########### - # preliminary tests - # adding existing returns exactly the existing + # Preliminary tests. + # Adding existing returns exactly the existing. sma = Submodule.add(rwrepo, sm.name, sm.path) assert sma.path == sm.path - # no url and no module at path fails + # No url and no module at path fails. self.assertRaises(ValueError, Submodule.add, rwrepo, "newsubm", "pathtorepo", url=None) # CONTINUE UPDATE ################# - # lets update it - its a recursive one too + # Let's update it - it's a recursive one too. newdir = osp.join(sm.abspath, "dir") os.makedirs(newdir) - # update fails if the path already exists non-empty + # Update fails if the path already exists non-empty. self.assertRaises(OSError, sm.update) os.rmdir(newdir) - # dry-run does nothing + # Dry-run does nothing. sm.update(dry_run=True, progress=prog) assert not sm.module_exists() assert sm.update() is sm - sm_repopath = sm.path # cache for later + sm_repopath = sm.path # Cache for later. assert sm.module_exists() assert isinstance(sm.module(), git.Repo) assert sm.module().working_tree_dir == sm.abspath # INTERLEAVE ADD TEST ##################### - # url must match the one in the existing repository ( if submodule name suggests a new one ) - # or we raise + # url must match the one in the existing repository (if submodule name suggests a new one) + # or we raise. self.assertRaises( ValueError, Submodule.add, @@ -213,53 +214,53 @@ def _do_base_tests(self, rwrepo): # CONTINUE UPDATE ################# - # we should have setup a tracking branch, which is also active + # We should have setup a tracking branch, which is also active. assert sm.module().head.ref.tracking_branch() is not None - # delete the whole directory and re-initialize + # Delete the whole directory and re-initialize. assert len(sm.children()) != 0 # shutil.rmtree(sm.abspath) sm.remove(force=True, configuration=False) assert len(sm.children()) == 0 - # dry-run does nothing + # Dry-run does nothing. sm.update(dry_run=True, recursive=False, progress=prog) assert len(sm.children()) == 0 sm.update(recursive=False) assert len(list(rwrepo.iter_submodules())) == 2 - assert len(sm.children()) == 1 # its not checked out yet + assert len(sm.children()) == 1 # It's not checked out yet. csm = sm.children()[0] assert not csm.module_exists() csm_repopath = csm.path - # adjust the path of the submodules module to point to the local destination + # Adjust the path of the submodules module to point to the local destination. new_csmclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path, csm.path)) with csm.config_writer() as writer: writer.set_value("url", new_csmclone_path) assert csm.url == new_csmclone_path - # dry-run does nothing + # Dry-run does nothing. assert not csm.module_exists() sm.update(recursive=True, dry_run=True, progress=prog) assert not csm.module_exists() - # update recursively again + # Update recursively again. sm.update(recursive=True) assert csm.module_exists() - # tracking branch once again + # Tracking branch once again. assert csm.module().head.ref.tracking_branch() is not None - # this flushed in a sub-submodule + # This flushed in a sub-submodule. assert len(list(rwrepo.iter_submodules())) == 2 - # reset both heads to the previous version, verify that to_latest_revision works + # Reset both heads to the previous version, verify that to_latest_revision works. smods = (sm.module(), csm.module()) for repo in smods: repo.head.reset("HEAD~2", working_tree=1) # END for each repo to reset - # dry run does nothing + # Dry-run does nothing. self.assertRaises( RepositoryDirtyError, sm.update, @@ -279,89 +280,89 @@ def _do_base_tests(self, rwrepo): # END for each repo to check del smods - # if the head is detached, it still works ( but warns ) + # If the head is detached, it still works (but warns). smref = sm.module().head.ref sm.module().head.ref = "HEAD~1" - # if there is no tracking branch, we get a warning as well + # If there is no tracking branch, we get a warning as well. csm_tracking_branch = csm.module().head.ref.tracking_branch() csm.module().head.ref.set_tracking_branch(None) sm.update(recursive=True, to_latest_revision=True) # to_latest_revision changes the child submodule's commit, it needs an - # update now + # update now. csm.set_parent_commit(csm.repo.head.commit) - # undo the changes + # Undo the changes. sm.module().head.ref = smref csm.module().head.ref.set_tracking_branch(csm_tracking_branch) # REMOVAL OF REPOSITORY ####################### - # must delete something + # Must delete something. self.assertRaises(ValueError, csm.remove, module=False, configuration=False) # module() is supposed to point to gitdb, which has a child-submodule whose URL is still pointing - # to GitHub. To save time, we will change it to + # to GitHub. To save time, we will change it to: csm.set_parent_commit(csm.repo.head.commit) with csm.config_writer() as cw: cw.set_value("url", self._small_repo_url()) csm.repo.index.commit("adjusted URL to point to local source, instead of the internet") # We have modified the configuration, hence the index is dirty, and the - # deletion will fail - # NOTE: As we did a few updates in the meanwhile, the indices were reset - # Hence we create some changes + # deletion will fail. + # NOTE: As we did a few updates in the meanwhile, the indices were reset. + # Hence we create some changes. csm.set_parent_commit(csm.repo.head.commit) with sm.config_writer() as writer: writer.set_value("somekey", "somevalue") with csm.config_writer() as writer: writer.set_value("okey", "ovalue") self.assertRaises(InvalidGitRepositoryError, sm.remove) - # if we remove the dirty index, it would work + # If we remove the dirty index, it would work. sm.module().index.reset() - # still, we have the file modified + # Still, we have the file modified. self.assertRaises(InvalidGitRepositoryError, sm.remove, dry_run=True) sm.module().index.reset(working_tree=True) - # enforce the submodule to be checked out at the right spot as well. + # Enforce the submodule to be checked out at the right spot as well. csm.update() assert csm.module_exists() assert csm.exists() assert osp.isdir(csm.module().working_tree_dir) - # this would work + # This would work. assert sm.remove(force=True, dry_run=True) is sm assert sm.module_exists() sm.remove(force=True, dry_run=True) assert sm.module_exists() - # but ... we have untracked files in the child submodule + # But... we have untracked files in the child submodule. fn = join_path_native(csm.module().working_tree_dir, "newfile") with open(fn, "w") as fd: fd.write("hi") self.assertRaises(InvalidGitRepositoryError, sm.remove) - # forcibly delete the child repository + # Forcibly delete the child repository. prev_count = len(sm.children()) self.assertRaises(ValueError, csm.remove, force=True) # We removed sm, which removed all submodules. However, the instance we - # have still points to the commit prior to that, where it still existed + # have still points to the commit prior to that, where it still existed. csm.set_parent_commit(csm.repo.commit(), check=False) assert not csm.exists() assert not csm.module_exists() assert len(sm.children()) == prev_count - # now we have a changed index, as configuration was altered. - # fix this + # Now we have a changed index, as configuration was altered. + # Fix this. sm.module().index.reset(working_tree=True) - # now delete only the module of the main submodule + # Now delete only the module of the main submodule. assert sm.module_exists() sm.remove(configuration=False, force=True) assert sm.exists() assert not sm.module_exists() assert sm.config_reader().get_value("url") - # delete the rest + # Delete the rest. sm_path = sm.path sm.remove() assert not sm.exists() @@ -372,7 +373,7 @@ def _do_base_tests(self, rwrepo): # ADD NEW SUBMODULE ################### - # add a simple remote repo - trailing slashes are no problem + # Add a simple remote repo - trailing slashes are no problem. smid = "newsub" osmid = "othersub" nsm = Submodule.add( @@ -386,11 +387,11 @@ def _do_base_tests(self, rwrepo): assert nsm.name == smid assert nsm.module_exists() assert nsm.exists() - # its not checked out + # It's not checked out. assert not osp.isfile(join_path_native(nsm.module().working_tree_dir, Submodule.k_modules_file)) assert len(rwrepo.submodules) == 1 - # add another submodule, but into the root, not as submodule + # Add another submodule, but into the root, not as submodule. osm = Submodule.add(rwrepo, osmid, csm_repopath, new_csmclone_path, Submodule.k_head_default) assert osm != nsm assert osm.module_exists() @@ -399,28 +400,28 @@ def _do_base_tests(self, rwrepo): assert len(rwrepo.submodules) == 2 - # commit the changes, just to finalize the operation + # Commit the changes, just to finalize the operation. rwrepo.index.commit("my submod commit") assert len(rwrepo.submodules) == 2 - # needs update as the head changed, it thinks its in the history - # of the repo otherwise + # Needs update, as the head changed. It thinks it's in the history + # of the repo otherwise. nsm.set_parent_commit(rwrepo.head.commit) osm.set_parent_commit(rwrepo.head.commit) # MOVE MODULE ############# - # invalid input + # Invalid input. self.assertRaises(ValueError, nsm.move, "doesntmatter", module=False, configuration=False) - # renaming to the same path does nothing + # Renaming to the same path does nothing. assert nsm.move(sm_path) is nsm - # rename a module - nmp = join_path_native("new", "module", "dir") + "/" # new module path + # Rename a module. + nmp = join_path_native("new", "module", "dir") + "/" # New module path. pmp = nsm.path assert nsm.move(nmp) is nsm - nmp = nmp[:-1] # cut last / + nmp = nmp[:-1] # Cut last / nmpl = to_native_path_linux(nmp) assert nsm.path == nmpl assert rwrepo.submodules[0].path == nmpl @@ -431,14 +432,14 @@ def _do_base_tests(self, rwrepo): self.assertRaises(ValueError, nsm.move, mpath) os.remove(absmpath) - # now it works, as we just move it back + # Now it works, as we just move it back. nsm.move(pmp) assert nsm.path == pmp assert rwrepo.submodules[0].path == pmp # REMOVE 'EM ALL ################ - # if a submodule's repo has no remotes, it can't be added without an explicit url + # If a submodule's repo has no remotes, it can't be added without an explicit url. osmod = osm.module() osm.remove(module=False) @@ -448,7 +449,7 @@ def _do_base_tests(self, rwrepo): self.assertRaises(ValueError, Submodule.add, rwrepo, osmid, csm_repopath, url=None) # END handle bare mode - # Error if there is no submodule file here + # Error if there is no submodule file here. self.assertRaises( IOError, Submodule._config_parser, @@ -487,11 +488,11 @@ def test_base_bare(self, rwrepo): ) @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): - # Can query everything without problems + # Can query everything without problems. rm = RootModule(self.rorepo) assert rm.module() is self.rorepo - # try attributes + # Try attributes. rm.binsha rm.mode rm.path @@ -505,110 +506,110 @@ def test_root_module(self, rwrepo): with rm.config_writer(): pass - # deep traversal gitdb / async + # Deep traversal gitdb / async. rsmsp = [sm.path for sm in rm.traverse()] - assert len(rsmsp) >= 2 # gitdb and async [and smmap], async being a child of gitdb + assert len(rsmsp) >= 2 # gitdb and async [and smmap], async being a child of gitdb. - # cannot set the parent commit as root module's path didn't exist + # Cannot set the parent commit as root module's path didn't exist. self.assertRaises(ValueError, rm.set_parent_commit, "HEAD") # TEST UPDATE ############# - # setup commit which remove existing, add new and modify existing submodules + # Set up a commit that removes existing, adds new and modifies existing submodules. rm = RootModule(rwrepo) assert len(rm.children()) == 1 - # modify path without modifying the index entry - # ( which is what the move method would do properly ) + # Modify path without modifying the index entry. + # (Which is what the move method would do properly.) # ================================================== sm = rm.children()[0] pp = "path/prefix" fp = join_path_native(pp, sm.path) prep = sm.path - assert not sm.module_exists() # was never updated after rwrepo's clone + assert not sm.module_exists() # It was never updated after rwrepo's clone. - # assure we clone from a local source + # Ensure we clone from a local source. with sm.config_writer() as writer: writer.set_value("url", Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path))) - # dry-run does nothing + # Dry-run does nothing. sm.update(recursive=False, dry_run=True, progress=prog) assert not sm.module_exists() sm.update(recursive=False) assert sm.module_exists() with sm.config_writer() as writer: - writer.set_value("path", fp) # change path to something with prefix AFTER url change + writer.set_value("path", fp) # Change path to something with prefix AFTER url change. - # update doesn't fail, because list_items ignores the wrong path in such situations. + # Update doesn't fail, because list_items ignores the wrong path in such situations. rm.update(recursive=False) - # move it properly - doesn't work as it its path currently points to an indexentry - # which doesn't exist ( move it to some path, it doesn't matter here ) + # Move it properly - doesn't work as it its path currently points to an indexentry + # which doesn't exist (move it to some path, it doesn't matter here). self.assertRaises(InvalidGitRepositoryError, sm.move, pp) - # reset the path(cache) to where it was, now it works + # Reset the path(cache) to where it was, now it works. sm.path = prep - sm.move(fp, module=False) # leave it at the old location + sm.move(fp, module=False) # Leave it at the old location. assert not sm.module_exists() - cpathchange = rwrepo.index.commit("changed sm path") # finally we can commit + cpathchange = rwrepo.index.commit("changed sm path") # Finally we can commit. - # update puts the module into place + # Update puts the module into place. rm.update(recursive=False, progress=prog) sm.set_parent_commit(cpathchange) assert sm.module_exists() - # add submodule - # ================ + # Add submodule. + # ============== nsmn = "newsubmodule" nsmp = "submrepo" subrepo_url = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0], rsmsp[1])) nsm = Submodule.add(rwrepo, nsmn, nsmp, url=subrepo_url) - csmadded = rwrepo.index.commit("Added submodule").hexsha # make sure we don't keep the repo reference + csmadded = rwrepo.index.commit("Added submodule").hexsha # Make sure we don't keep the repo reference. nsm.set_parent_commit(csmadded) assert nsm.module_exists() - # in our case, the module should not exist, which happens if we update a parent - # repo and a new submodule comes into life + # In our case, the module should not exist, which happens if we update a parent + # repo and a new submodule comes into life. nsm.remove(configuration=False, module=True) assert not nsm.module_exists() and nsm.exists() - # dry-run does nothing + # Dry-run does nothing. rm.update(recursive=False, dry_run=True, progress=prog) - # otherwise it will work + # Otherwise it will work. rm.update(recursive=False, progress=prog) assert nsm.module_exists() - # remove submodule - the previous one + # Remove submodule - the previous one. # ==================================== sm.set_parent_commit(csmadded) smp = sm.abspath assert not sm.remove(module=False).exists() - assert osp.isdir(smp) # module still exists + assert osp.isdir(smp) # Module still exists. csmremoved = rwrepo.index.commit("Removed submodule") - # an update will remove the module - # not in dry_run + # An update will remove the module. + # Not in dry_run. rm.update(recursive=False, dry_run=True, force_remove=True) assert osp.isdir(smp) - # when removing submodules, we may get new commits as nested submodules are auto-committing changes + # When removing submodules, we may get new commits as nested submodules are auto-committing changes # to allow deletions without force, as the index would be dirty otherwise. # QUESTION: Why does this seem to work in test_git_submodule_compatibility() ? self.assertRaises(InvalidGitRepositoryError, rm.update, recursive=False, force_remove=False) rm.update(recursive=False, force_remove=True) assert not osp.isdir(smp) - # 'apply work' to the nested submodule and assure this is not removed/altered during updates - # Need to commit first, otherwise submodule.update wouldn't have a reason to change the head + # 'Apply work' to the nested submodule and ensure this is not removed/altered during updates + # Need to commit first, otherwise submodule.update wouldn't have a reason to change the head. touch(osp.join(nsm.module().working_tree_dir, "new-file")) - # We cannot expect is_dirty to even run as we wouldn't reset a head to the same location + # We cannot expect is_dirty to even run as we wouldn't reset a head to the same location. assert nsm.module().head.commit.hexsha == nsm.hexsha nsm.module().index.add([nsm]) nsm.module().index.commit("added new file") - rm.update(recursive=False, dry_run=True, progress=prog) # would not change head, and thus doesn't fail - # Everything we can do from now on will trigger the 'future' check, so no is_dirty() check will even run - # This would only run if our local branch is in the past and we have uncommitted changes + rm.update(recursive=False, dry_run=True, progress=prog) # Would not change head, and thus doesn't fail. + # Everything we can do from now on will trigger the 'future' check, so no is_dirty() check will even run. + # This would only run if our local branch is in the past and we have uncommitted changes. prev_commit = nsm.module().head.commit rm.update(recursive=False, dry_run=False, progress=prog) @@ -618,10 +619,10 @@ def test_root_module(self, rwrepo): rm.update(recursive=True, progress=prog, force_reset=True) assert prev_commit != nsm.module().head.commit, "head changed, as the remote url and its commit changed" - # change url ... - # =============== - # ... to the first repository, this way we have a fast checkout, and a completely different - # repository at the different url + # Change url... + # ============= + # ...to the first repository. This way we have a fast checkout, and a completely different + # repository at the different url. nsm.set_parent_commit(csmremoved) nsmurl = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0])) with nsm.config_writer() as writer: @@ -629,7 +630,7 @@ def test_root_module(self, rwrepo): csmpathchange = rwrepo.index.commit("changed url") nsm.set_parent_commit(csmpathchange) - # Now nsm head is in the future of the tracked remote branch + # Now nsm head is in the future of the tracked remote branch. prev_commit = nsm.module().head.commit # dry-run does nothing rm.update(recursive=False, dry_run=True, progress=prog) @@ -641,16 +642,16 @@ def test_root_module(self, rwrepo): assert len(rwrepo.submodules) == 1 assert not rwrepo.submodules[0].children()[0].module_exists(), "nested submodule should not be checked out" - # add the submodule's changed commit to the index, which is what the - # user would do - # beforehand, update our instance's binsha with the new one + # Add the submodule's changed commit to the index, which is what the + # user would do. + # Beforehand, update our instance's binsha with the new one. nsm.binsha = nsm.module().head.commit.binsha rwrepo.index.add([nsm]) - # change branch - # ================= - # we only have one branch, so we switch to a virtual one, and back - # to the current one to trigger the difference + # Change branch. + # ============== + # We only have one branch, so we switch to a virtual one, and back + # to the current one to trigger the difference. cur_branch = nsm.branch nsmm = nsm.module() prev_commit = nsmm.head.commit @@ -661,35 +662,35 @@ def test_root_module(self, rwrepo): nsm.set_parent_commit(csmbranchchange) # END for each branch to change - # Lets remove our tracking branch to simulate some changes + # Let's remove our tracking branch to simulate some changes. nsmmh = nsmm.head - assert nsmmh.ref.tracking_branch() is None # never set it up until now + assert nsmmh.ref.tracking_branch() is None # Never set it up until now. assert not nsmmh.is_detached - # dry run does nothing + # Dry-run does nothing. rm.update(recursive=False, dry_run=True, progress=prog) assert nsmmh.ref.tracking_branch() is None - # the real thing does + # The real thing does. rm.update(recursive=False, progress=prog) assert nsmmh.ref.tracking_branch() is not None assert not nsmmh.is_detached - # recursive update + # Recursive update. # ================= - # finally we recursively update a module, just to run the code at least once - # remove the module so that it has more work - assert len(nsm.children()) >= 1 # could include smmap + # Finally we recursively update a module, just to run the code at least once + # remove the module so that it has more work. + assert len(nsm.children()) >= 1 # Could include smmap. assert nsm.exists() and nsm.module_exists() and len(nsm.children()) >= 1 - # assure we pull locally only + # Ensure we pull locally only. nsmc = nsm.children()[0] with nsmc.config_writer() as writer: writer.set_value("url", subrepo_url) - rm.update(recursive=True, progress=prog, dry_run=True) # just to run the code + rm.update(recursive=True, progress=prog, dry_run=True) # Just to run the code. rm.update(recursive=True, progress=prog) - # gitdb: has either 1 or 2 submodules depending on the version + # gitdb: has either 1 or 2 submodules depending on the version. assert len(nsm.children()) >= 1 and nsmc.module_exists() @with_rw_repo(k_no_subm_tag, bare=False) @@ -742,7 +743,7 @@ def test_list_only_valid_submodules(self, rwdir): assert len(repo.submodules) == 1 - # Delete the directory from submodule + # Delete the directory from submodule. submodule_path = osp.join(repo_path, "module") shutil.rmtree(submodule_path) repo.git.add([submodule_path]) @@ -800,13 +801,13 @@ def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): smm.git.add(Git.polish_url(fp)) smm.git.commit(m="new file added") - # submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule - # object pointing to the new submodule commit + # Submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule + # object pointing to the new submodule commit. sm_too = parent.submodules["module_moved"] assert parent.head.commit.tree[sm.path].binsha == sm.binsha assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one" - added_bies = parent.index.add([sm]) # added base-index-entries + added_bies = parent.index.add([sm]) # Added base-index-entries. assert len(added_bies) == 1 parent.index.commit("add same submodule entry") commit_sm = parent.head.commit.tree[sm.path] @@ -840,8 +841,8 @@ def assert_exists(sm, value=True): # end - # As git is backwards compatible itself, it would still recognize what we do here ... unless we really - # muss it up. That's the only reason why the test is still here ... . + # As git is backwards compatible itself, it would still recognize what we do here... unless we really + # muss it up. That's the only reason why the test is still here... assert len(parent.git.submodule().splitlines()) == 1 module_repo_path = osp.join(sm.module().working_tree_dir, ".git") @@ -855,12 +856,12 @@ def assert_exists(sm, value=True): assert find_submodule_git_dir(module_repo_path) is not None, "module pointed to by .git file must be valid" # end verify submodule 'style' - # test move + # Test move. new_sm_path = join_path_native("submodules", "one") sm.move(new_sm_path) assert_exists(sm) - # Add additional submodule level + # Add additional submodule level. csm = sm.module().create_submodule( "nested-submodule", join_path_native("nested-submodule", "working-tree"), @@ -870,21 +871,21 @@ def assert_exists(sm, value=True): sm_head_commit = sm.module().commit() assert_exists(csm) - # Fails because there are new commits, compared to the remote we cloned from + # Fails because there are new commits, compared to the remote we cloned from. self.assertRaises(InvalidGitRepositoryError, sm.remove, dry_run=True) assert_exists(sm) assert sm.module().commit() == sm_head_commit assert_exists(csm) - # rename nested submodule - # This name would move itself one level deeper - needs special handling internally + # Rename nested submodule. + # This name would move itself one level deeper - needs special handling internally. new_name = csm.name + "/mine" assert csm.rename(new_name).name == new_name assert_exists(csm) assert csm.repo.is_dirty(index=True, working_tree=False), "index must contain changed .gitmodules file" csm.repo.index.commit("renamed module") - # keep_going evaluation + # keep_going evaluation. rsm = parent.submodule_update() assert_exists(sm) assert_exists(csm) @@ -944,11 +945,11 @@ def test_remove_norefs(self, rwdir): parent.index.commit("Added submodule") assert sm.repo is parent # yoh was surprised since expected sm repo!! - # so created a new instance for submodule + # So created a new instance for submodule. smrepo = git.Repo(osp.join(rwdir, "parent", sm.path)) - # Adding a remote without fetching so would have no references + # Adding a remote without fetching so would have no references. smrepo.create_remote("special", "git@server-shouldnotmatter:repo.git") - # And we should be able to remove it just fine + # And we should be able to remove it just fine. sm.remove() assert not sm.exists() @@ -977,8 +978,8 @@ def test_rename(self, rwdir): @with_rw_directory def test_branch_renames(self, rw_dir): - # Setup initial sandbox: - # parent repo has one submodule, which has all the latest changes + # Set up initial sandbox: + # The parent repo has one submodule, which has all the latest changes. source_url = self._small_repo_url() sm_source_repo = git.Repo.clone_from(source_url, osp.join(rw_dir, "sm-source"), b="master") parent_repo = git.Repo.init(osp.join(rw_dir, "parent")) @@ -991,20 +992,20 @@ def test_branch_renames(self, rw_dir): parent_repo.index.commit("added submodule") assert sm.exists() - # Create feature branch with one new commit in submodule source + # Create feature branch with one new commit in submodule source. sm_fb = sm_source_repo.create_head("feature") sm_fb.checkout() new_file = touch(osp.join(sm_source_repo.working_tree_dir, "new-file")) sm_source_repo.index.add([new_file]) sm.repo.index.commit("added new file") - # change designated submodule checkout branch to the new upstream feature branch + # Change designated submodule checkout branch to the new upstream feature branch. with sm.config_writer() as smcw: smcw.set_value("branch", sm_fb.name) assert sm.repo.is_dirty(index=True, working_tree=False) sm.repo.index.commit("changed submodule branch to '%s'" % sm_fb) - # verify submodule update with feature branch that leaves currently checked out branch in it's past + # Verify submodule update with feature branch that leaves currently checked out branch in it's past. sm_mod = sm.module() prev_commit = sm_mod.commit() assert sm_mod.head.ref.name == "master" @@ -1016,22 +1017,22 @@ def test_branch_renames(self, rw_dir): assert sm_mod.head.ref.name == sm_fb.name assert sm_mod.commit() == sm_fb.commit - # Create new branch which is in our past, and thus seemingly unrelated to the currently checked out one - # To make it even 'harder', we shall fork and create a new commit + # Create new branch which is in our past, and thus seemingly unrelated to the currently checked out one. + # To make it even 'harder', we shall fork and create a new commit. sm_pfb = sm_source_repo.create_head("past-feature", commit="HEAD~20") sm_pfb.checkout() sm_source_repo.index.add([touch(osp.join(sm_source_repo.working_tree_dir, "new-file"))]) sm_source_repo.index.commit("new file added, to past of '%r'" % sm_fb) - # Change designated submodule checkout branch to a new commit in its own past + # Change designated submodule checkout branch to a new commit in its own past. with sm.config_writer() as smcw: smcw.set_value("branch", sm_pfb.path) sm.repo.index.commit("changed submodule branch to '%s'" % sm_pfb) - # Test submodule updates - must fail if submodule is dirty + # Test submodule updates - must fail if submodule is dirty. touch(osp.join(sm_mod.working_tree_dir, "unstaged file")) # This doesn't fail as our own submodule binsha didn't change, and the reset is only triggered if - # to latest revision is True. + # to_latest_revision is True. parent_repo.submodule_update(to_latest_revision=False) assert sm_mod.head.ref.name == sm_pfb.name, "should have been switched to past head" assert sm_mod.commit() == sm_fb.commit, "Head wasn't reset" diff --git a/test/test_tree.py b/test/test_tree.py index c5ac8d539..5fc98e40c 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -14,14 +14,14 @@ class TestTree(TestBase): def test_serializable(self): - # tree at the given commit contains a submodule as well + # Tree at the given commit contains a submodule as well. roottree = self.rorepo.tree("6c1faef799095f3990e9970bc2cb10aa0221cf9c") for item in roottree.traverse(ignore_self=False): if item.type != Tree.type: continue # END skip non-trees tree = item - # trees have no dict + # Trees have no dict. self.assertRaises(AttributeError, setattr, tree, "someattr", 1) orig_data = tree.data_stream.read() @@ -36,7 +36,7 @@ def test_serializable(self): testtree._deserialize(stream) assert testtree._cache == orig_cache - # replaces cache, but we make sure of it + # Replaces cache, but we make sure of it. del testtree._cache testtree._deserialize(stream) # END for each item in tree @@ -54,29 +54,29 @@ def test_traverse(self): # END for each object assert all_items == root.list_traverse() - # limit recursion level to 0 - should be same as default iteration + # Limit recursion level to 0 - should be same as default iteration. assert all_items assert "CHANGES" in root assert len(list(root)) == len(list(root.traverse(depth=1))) - # only choose trees + # Only choose trees. trees_only = lambda i, d: i.type == "tree" trees = list(root.traverse(predicate=trees_only)) assert len(trees) == len([i for i in root.traverse() if trees_only(i, 0)]) - # test prune + # Test prune. lib_folder = lambda t, d: t.path == "lib" pruned_trees = list(root.traverse(predicate=trees_only, prune=lib_folder)) assert len(pruned_trees) < len(trees) - # trees and blobs + # Trees and blobs. assert len(set(trees) | set(root.trees)) == len(trees) assert len({b for b in root if isinstance(b, Blob)} | set(root.blobs)) == len(root.blobs) subitem = trees[0][0] assert "/" in subitem.path assert subitem.name == osp.basename(subitem.path) - # assure that at some point the traversed paths have a slash in them + # Check that at some point the traversed paths have a slash in them. found_slash = False for item in root.traverse(): assert osp.isabs(item.abspath) @@ -84,8 +84,8 @@ def test_traverse(self): found_slash = True # END check for slash - # slashes in paths are supported as well - # NOTE: on py3, / doesn't work with strings anymore ... + # Slashes in paths are supported as well. + # NOTE: On Python 3, / doesn't work with strings anymore... assert root[item.path] == item == root / item.path # END for each item assert found_slash From cf9243a265a07b834e19974cd0c8d3a290379da7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 22 Oct 2023 20:20:14 -0400 Subject: [PATCH 0814/1790] Add a module docstring to tstrunner.py It benefits from a docstring because it is not referenced in code or documentation, and its purpose is/was otherwise unclear. The docstring includes a reference to #1188 for more information. --- test/tstrunner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/tstrunner.py b/test/tstrunner.py index 441050c68..8613538eb 100644 --- a/test/tstrunner.py +++ b/test/tstrunner.py @@ -1,3 +1,5 @@ +"""Hook for MonkeyType (see PR #1188).""" + import unittest loader = unittest.TestLoader() From b5c3ca4bd29ee42d6a9f51ff21645e9165c7f0e7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 22 Oct 2023 20:26:12 -0400 Subject: [PATCH 0815/1790] Slightly improve readability of installation-test strings This modifies content of the strings themselves, but only slightly. - Make an exception message more readable. - Improve spacing of the "python -c" script string in the sys.path test. --- test/test_installation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_installation.py b/test/test_installation.py index 3a0e43973..e7774d29d 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -45,7 +45,7 @@ def test_installation(self, rw_dir): self.assertEqual( 0, result.returncode, - msg=result.stderr or result.stdout or "Selftest failed", + msg=result.stderr or result.stdout or "Self-test failed", ) result = subprocess.run( @@ -63,7 +63,7 @@ def test_installation(self, rw_dir): # by inserting its location into PYTHONPATH or otherwise patched into # sys.path, make sure it is not wrongly inserted as the *first* entry. result = subprocess.run( - [self.python, "-c", "import sys;import git; print(sys.path)"], + [self.python, "-c", "import sys; import git; print(sys.path)"], stdout=subprocess.PIPE, cwd=self.sources, ) From f444470bc2614b6b148ec4d8761ac804f72343b3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 22 Oct 2023 21:09:41 -0400 Subject: [PATCH 0816/1790] More wording improvements (in git module) This futher improves some wording in docstrings and comments in a handful of places within the git module. --- git/cmd.py | 2 +- git/config.py | 13 +++++-------- git/diff.py | 4 ++-- git/index/base.py | 8 ++++---- git/objects/submodule/root.py | 6 +++--- git/objects/tree.py | 4 ++-- git/repo/base.py | 2 +- git/util.py | 10 +++++----- 8 files changed, 23 insertions(+), 26 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 384a39077..de7d34bf5 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -610,7 +610,7 @@ class CatFileContentStream: stream once our sized content region is empty. If not all data are read to the end of the object's lifetime, we read the - rest to assure the underlying stream continues to work. + rest to ensure the underlying stream continues to work. """ __slots__: Tuple[str, ...] = ("_stream", "_nbr", "_size") diff --git a/git/config.py b/git/config.py index 31b8a665d..f0002658b 100644 --- a/git/config.py +++ b/git/config.py @@ -56,7 +56,7 @@ T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool) if sys.version_info[:3] < (3, 7, 2): - # typing.Ordereddict not added until py 3.7.2 + # typing.Ordereddict not added until Python 3.7.2. from collections import OrderedDict OrderedDict_OMD = OrderedDict @@ -73,13 +73,10 @@ log = logging.getLogger("git.config") log.addHandler(logging.NullHandler()) -# invariants -# represents the configuration level of a configuration file - +# The configuration level of a configuration file. CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") - # Section pattern to detect conditional includes. # https://git-scm.com/docs/git-config#_conditional_includes CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") @@ -603,7 +600,7 @@ def read(self) -> None: # type: ignore[override] files_to_read = [self._file_or_files] else: # for lists or tuples files_to_read = list(self._file_or_files) - # end assure we have a copy of the paths to handle + # end ensure we have a copy of the paths to handle seen = set(files_to_read) num_read_include_files = 0 @@ -612,7 +609,7 @@ def read(self) -> None: # type: ignore[override] file_ok = False if hasattr(file_path, "seek"): - # Must be a file objectfile-object. + # Must be a file-object. file_path = cast(IO[bytes], file_path) # TODO: Replace with assert to narrow type, once sure. self._read(file_path, file_path.name) else: @@ -626,7 +623,7 @@ def read(self) -> None: # type: ignore[override] continue # Read includes and append those that we didn't handle yet. - # We expect all paths to be normalized and absolute (and will assure that is the case). + # We expect all paths to be normalized and absolute (and will ensure that is the case). if self._has_includes(): for _, include_path in self._included_paths(): if include_path.startswith("~"): diff --git a/git/diff.py b/git/diff.py index d5740a7fc..08ae09444 100644 --- a/git/diff.py +++ b/git/diff.py @@ -122,8 +122,8 @@ def diff( the respective tree. * If :class:`Index `, it will be compared against the index. * If :attr:`git.NULL_TREE`, it will compare against the empty tree. - * It defaults to :class:`Index ` to assure the method will - not by-default fail on bare repositories. + * It defaults to :class:`Index ` so that the method will not + by default fail on bare repositories. :param paths: This a list of paths or a single path to limit the diff to. It will only diff --git a/git/index/base.py b/git/index/base.py index 4f1b6c469..a43afb320 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -340,9 +340,9 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile resolve more cases in a commonly correct manner. Specify trivial=True as kwarg to override that. - As the underlying git-read-tree command takes into account the current index, - it will be temporarily moved out of the way to assure there are no unsuspected - interferences. + As the underlying git-read-tree command takes into account the current + index, it will be temporarily moved out of the way to prevent any unexpected + interference. """ if len(treeish) == 0 or len(treeish) > 3: raise ValueError("Please specify between 1 and 3 treeish, got %i" % len(treeish)) @@ -366,7 +366,7 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile # Move current index out of the way - otherwise the merge may fail # as it considers existing entries. Moving it essentially clears the index. # Unfortunately there is no 'soft' way to do it. - # The TemporaryFileSwap assure the original file get put back. + # The TemporaryFileSwap ensures the original file gets put back. stack.enter_context(TemporaryFileSwap(join_path_native(repo.git_dir, "index"))) repo.git.read_tree(*arg_list, **kwargs) diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index b3c06ce9a..bc61f4ac2 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -133,7 +133,7 @@ def update( if progress is None: progress = RootUpdateProgress() - # END assure progress is set + # END ensure progress is set prefix = "" if dry_run: @@ -300,7 +300,7 @@ def update( smr.rename(orig_name) # Early on, we verified that the our current tracking branch - # exists in the remote. Now we have to assure that the + # exists in the remote. Now we have to ensure that the # sha we point to is still contained in the new remote # tracking branch. smsha = sm.binsha @@ -367,7 +367,7 @@ def update( except OSError: # ...or reuse the existing one. tbr = git.Head(smm, sm.branch_path) - # END assure tracking branch exists + # END ensure tracking branch exists tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name)) # NOTE: All head-resetting is done in the base implementation of update diff --git a/git/objects/tree.py b/git/objects/tree.py index c91c01807..708ab3edd 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -180,8 +180,8 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod return self def add_unchecked(self, binsha: bytes, mode: int, name: str) -> None: - """Add the given item to the tree. Its correctness is assumed, which - puts the caller into responsibility to assure the input is correct. + """Add the given item to the tree. Its correctness is assumed, so it is the + caller's responsibility to ensure that the input is correct. For more information on the parameters, see :meth:`add`. diff --git a/git/repo/base.py b/git/repo/base.py index 2694282d0..5d0a06198 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1448,7 +1448,7 @@ def archive( path = cast(Union[PathLike, List[PathLike], Tuple[PathLike, ...]], path) if not isinstance(path, (tuple, list)): path = [path] - # end assure paths is list + # end ensure paths is list (or tuple) self.git.archive("--", treeish, *path, **kwargs) return self diff --git a/git/util.py b/git/util.py index 4e96f7394..adbc8146f 100644 --- a/git/util.py +++ b/git/util.py @@ -283,7 +283,7 @@ def to_native_path_linux(path: PathLike) -> str: def join_path_native(a: PathLike, *p: PathLike) -> PathLike: R"""Like join_path, but makes sure an OS native path is returned. - This is only needed to play it safe on Windows and to assure nice paths that only + This is only needed to play it safe on Windows and to ensure nice paths that only use '\'. """ return to_native_path(join_path(a, *p)) @@ -1185,8 +1185,8 @@ def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: class Iterable(metaclass=IterableClassWatcher): - """Defines an interface for iterable items which is to assure a uniform - way to retrieve and iterate items within the git repository.""" + """Defines an interface for iterable items, so there is a uniform way to retrieve + and iterate items within the git repository.""" __slots__ = () @@ -1221,8 +1221,8 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: @runtime_checkable class IterableObj(Protocol): - """Defines an interface for iterable items which is to assure a uniform - way to retrieve and iterate items within the git repository. + """Defines an interface for iterable items, so there is a uniform way to retrieve + and iterate items within the git repository. Subclasses = [Submodule, Commit, Reference, PushInfo, FetchInfo, Remote] """ From db317f1086b6886cbd43605d93b0f7f28a53f917 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 23 Oct 2023 01:59:18 -0400 Subject: [PATCH 0817/1790] Change :returns: to :return: :return: was used extensively, while :returns: only appeared in two docstrings. --- git/diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/diff.py b/git/diff.py index 08ae09444..829d53924 100644 --- a/git/diff.py +++ b/git/diff.py @@ -467,7 +467,7 @@ def rename_to(self) -> Optional[str]: @property def renamed(self) -> bool: """ - :returns: True if the blob of our diff has been renamed + :return: True if the blob of our diff has been renamed :note: This property is deprecated. Please use the :attr:`renamed_file` property instead. @@ -476,7 +476,7 @@ def renamed(self) -> bool: @property def renamed_file(self) -> bool: - """:returns: True if the blob of our diff has been renamed""" + """:return: True if the blob of our diff has been renamed""" return self.rename_from != self.rename_to @classmethod From 720e4bbf2bfe2dc5dcf8c2fe68a30c797231120c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 23 Oct 2023 02:02:58 -0400 Subject: [PATCH 0818/1790] Fix TestBigRepoR.setUp info message Changes that fix the message itself: - Add a missing space between words (two parts were concatenated, with no space at the edge of either). - Capitalize "GitPython" since that is the repo and project name. Changes that improve how the message is produced: - Make the entire literal part of the string the format string, instead of formatting the first part and concatenating the second part. - Pass the format string and k_env_git_repo variable as separate arguments to logging.info, so the logging machinery takes care of substituting it for %s, rather than doing the substitution. --- test/performance/lib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/performance/lib.py b/test/performance/lib.py index c793d771f..d1f0523f5 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -45,8 +45,9 @@ def setUp(self): repo_path = os.environ.get(k_env_git_repo) if repo_path is None: logging.info( - ("You can set the %s environment variable to a .git repository of" % k_env_git_repo) - + "your choice - defaulting to the gitpython repository" + "You can set the %s environment variable to a .git repository of your" + " choice - defaulting to the GitPython repository", + k_env_git_repo, ) repo_path = osp.dirname(__file__) # end set some repo path From faa19ac8bec46bb2d3dd26c9ea0e9dc1fb205de4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 00:05:56 -0400 Subject: [PATCH 0819/1790] Remove outdated git_daemon_launched Windows info This removes the Windows-specific information in the warning message in git_daemon_launched. After the associated functionality was updated in #1684 and the warning message was abridged accordingly, the functionality was updated again in #1697, causing the message to be outdated and no longer helpeful (since having git-daemon.exe in a PATH directory is no longer necessary or useful), without any corresponding change to the message. --- test/lib/helper.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index d415ba2e7..cef6779be 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -213,14 +213,6 @@ def git_daemon_launched(base_path, ip, port): and setting the environment variable GIT_PYTHON_TEST_GIT_DAEMON_PORT to """ ) - if is_win: - msg += textwrap.dedent( - R""" - - On Windows, - the `git-daemon.exe` must be in PATH. - For MINGW, look into \Git\mingw64\libexec\git-core\, but problems with paths might appear.""" - ) log.warning(msg, ex, ip, port, base_path, base_path, exc_info=1) yield # OK, assume daemon started manually. From bffc537d17a47c354d5fd36338ff7318a08995c2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 01:02:55 -0400 Subject: [PATCH 0820/1790] Revise docstrings/comments in test helpers --- test/fixtures/env_case.py | 2 +- test/lib/helper.py | 70 +++++++++++++++++++++------------------ test/performance/lib.py | 23 +++++-------- 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/test/fixtures/env_case.py b/test/fixtures/env_case.py index fe85ac41d..03b4df222 100644 --- a/test/fixtures/env_case.py +++ b/test/fixtures/env_case.py @@ -1,4 +1,4 @@ -# Steps 3 and 4 for test_it_avoids_upcasing_unrelated_environment_variable_names. +"""Steps 3 and 4 for test_it_avoids_upcasing_unrelated_environment_variable_names.""" import subprocess import sys diff --git a/test/lib/helper.py b/test/lib/helper.py index cef6779be..03ae62239 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -3,6 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + import contextlib from functools import wraps import gc @@ -65,9 +66,10 @@ def fixture(name): class StringProcessAdapter(object): + """Allows strings to be used as process objects returned by subprocess.Popen. - """Allows to use strings as Process object as returned by SubProcess.Popen. - Its tailored to work with the test system only""" + This is tailored to work with the test system only. + """ def __init__(self, input_string): self.stdout = io.BytesIO(input_string) @@ -86,7 +88,7 @@ def wait(self): def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the - test succeeds, but leave it otherwise to aid additional debugging""" + test succeeds, but leave it otherwise to aid additional debugging.""" @wraps(func) def wrapper(self): @@ -106,7 +108,7 @@ def wrapper(self): raise finally: # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as + # a Windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. gc.collect() @@ -117,8 +119,7 @@ def wrapper(self): def with_rw_repo(working_tree_ref, bare=False): - """ - Same as with_bare_repo, but clones the rorepo as non-bare repository, checking + """Same as with_bare_repo, but clones the rorepo as non-bare repository, checking out the working tree at the given working_tree_ref. This repository type is more costly due to the working copy checkout. @@ -199,7 +200,7 @@ def git_daemon_launched(base_path, ip, port): base_path=base_path, as_process=True, ) - # yes, I know ... fortunately, this is always going to work if sleep time is just large enough + # Yes, I know... fortunately, this is always going to work if sleep time is just large enough. time.sleep(0.5 * (1 + is_win)) except Exception as ex: msg = textwrap.dedent( @@ -225,33 +226,36 @@ def git_daemon_launched(base_path, ip, port): log.debug("Killing git-daemon...") gd.proc.kill() except Exception as ex: - ## Either it has died (and we're here), or it won't die, again here... + # Either it has died (and we're here), or it won't die, again here... log.debug("Hidden error while Killing git-daemon: %s", ex, exc_info=1) def with_rw_and_rw_remote_repo(working_tree_ref): - """ - Same as with_rw_repo, but also provides a writable remote repository from which the - rw_repo has been forked as well as a handle for a git-daemon that may be started to - run the remote_repo. - The remote repository was cloned as bare repository from the ro repo, whereas - the rw repo has a working tree and was cloned from the remote repository. + """Same as with_rw_repo, but also provides a writable remote repository from which + the rw_repo has been forked as well as a handle for a git-daemon that may be started + to run the remote_repo. - remote_repo has two remotes: origin and daemon_origin. One uses a local url, - the other uses a server url. The daemon setup must be done on system level - and should be an inetd service that serves tempdir.gettempdir() and all - directories in it. + The remote repository was cloned as bare repository from the ro repo, whereas the rw + repo has a working tree and was cloned from the remote repository. + + remote_repo has two remotes: origin and daemon_origin. One uses a local url, the + other uses a server url. The daemon setup must be done on system level and should be + an inetd service that serves tempdir.gettempdir() and all directories in it. The following sketch demonstrates this:: - rorepo ------> rw_remote_repo ------> rw_repo + + rorepo ------> rw_remote_repo ------> rw_repo The test case needs to support the following signature:: + def case(self, rw_repo, rw_daemon_repo) This setup allows you to test push and pull scenarios and hooks nicely. - See working dir info in with_rw_repo - :note: We attempt to launch our own invocation of git-daemon, which will be shutdown at the end of the test. + See working dir info in :func:`with_rw_repo`. + + :note: We attempt to launch our own invocation of git-daemon, which will be shut + down at the end of the test. """ from git import Git, Remote # To avoid circular deps. @@ -264,16 +268,16 @@ def remote_repo_creator(self): rw_repo_dir = tempfile.mktemp(prefix="daemon_cloned_repo-%s-" % func.__name__) rw_daemon_repo = self.rorepo.clone(rw_daemon_repo_dir, shared=True, bare=True) - # recursive alternates info ? + # Recursive alternates info? rw_repo = rw_daemon_repo.clone(rw_repo_dir, shared=True, bare=False, n=True) try: rw_repo.head.commit = working_tree_ref rw_repo.head.reference.checkout() - # prepare for git-daemon + # Prepare for git-daemon. rw_daemon_repo.daemon_export = True - # this thing is just annoying ! + # This thing is just annoying! with rw_daemon_repo.config_writer() as crw: section = "daemon" try: @@ -340,9 +344,7 @@ def remote_repo_creator(self): class TestBase(TestCase): - - """ - Base Class providing default functionality to all tests such as: + """Base class providing default functionality to all tests such as: - Utility functions provided by the TestCase base of the unittest method such as:: self.fail("todo") @@ -355,20 +357,20 @@ class TestBase(TestCase): The rorepo is in fact your current project's git repo. If you refer to specific shas for your objects, be sure you choose some that are part of the immutable portion - of the project history ( to assure tests don't fail for others ). + of the project history (so that tests don't fail for others). """ def _small_repo_url(self): - """:return" a path to a small, clonable repository""" + """:return: A path to a small, clonable repository""" from git.cmd import Git return Git.polish_url(osp.join(self.rorepo.working_tree_dir, "git/ext/gitdb/gitdb/ext/smmap")) @classmethod def setUpClass(cls): - """ - Dynamically add a read-only repository to our actual type. This way - each test type has its own repository + """Dynamically add a read-only repository to our actual type. + + This way, each test type has its own repository. """ from git import Repo @@ -383,7 +385,9 @@ def tearDownClass(cls): def _make_file(self, rela_path, data, repo=None): """ Create a file at the given path relative to our repository, filled - with the given data. Returns absolute path to created file. + with the given data. + + :return: An absolute path to the created file. """ repo = repo or self.rorepo abs_path = osp.join(repo.working_tree_dir, rela_path) diff --git a/test/performance/lib.py b/test/performance/lib.py index d1f0523f5..8c68db782 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -1,4 +1,5 @@ -"""Contains library functions""" +"""Support library for tests.""" + import logging import os import tempfile @@ -20,22 +21,16 @@ class TestBigRepoR(TestBase): - """TestCase providing access to readonly 'big' repositories using the following member variables: - * gitrorepo - - * Read-Only git repository - actually the repo of git itself - - * puregitrorepo + * gitrorepo: + Read-Only git repository - actually (by default) the repo of GitPython itself. - * As gitrepo, but uses pure python implementation + * puregitrorepo: + Like gitrorepo, but uses a pure Python implementation for its object database. """ - # { Invariants - # } END invariants - def setUp(self): try: super(TestBigRepoR, self).setUp() @@ -62,10 +57,10 @@ def tearDown(self): class TestBigRepoRW(TestBigRepoR): + """Like :class:`TestBigRepoR`, but provides a big repository that we can write to. - """As above, but provides a big repository that we can write to. - - Provides ``self.gitrwrepo`` and ``self.puregitrwrepo``""" + Provides ``self.gitrwrepo`` and ``self.puregitrwrepo``. + """ def setUp(self): self.gitrwrepo = None From 30f49d9deaa0489b42d93c96df584690b39aa79d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 01:10:55 -0400 Subject: [PATCH 0821/1790] Revise docstrings/comments in performance tests --- test/performance/test_commit.py | 20 ++++++++-------- test/performance/test_odb.py | 9 ++++---- test/performance/test_streams.py | 39 ++++++++++++++++---------------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index dbe2ad43e..fad0641be 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -1,8 +1,10 @@ -# test_performance.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Performance tests for commits (iteration, traversal, and serialization).""" + from io import BytesIO from time import time import sys @@ -19,7 +21,7 @@ def tearDown(self): gc.collect() - # ref with about 100 commits in its history + # ref with about 100 commits in its history. ref_100 = "0.1.6" def _query_commit_info(self, c): @@ -36,9 +38,9 @@ def test_iteration(self): no = 0 nc = 0 - # find the first commit containing the given path - always do a full - # iteration ( restricted to the path in question ), but in fact it should - # return quite a lot of commits, we just take one and hence abort the operation + # Find the first commit containing the given path. Always do a full iteration + # (restricted to the path in question). This should return quite a lot of + # commits. We just take one and hence abort the operation. st = time() for c in self.rorepo.iter_commits(self.ref_100): @@ -57,7 +59,7 @@ def test_iteration(self): ) def test_commit_traversal(self): - # bound to cat-file parsing performance + # Bound to cat-file parsing performance. nc = 0 st = time() for c in self.gitrorepo.commit().traverse(branch_first=False): @@ -71,7 +73,7 @@ def test_commit_traversal(self): ) def test_commit_iteration(self): - # bound to stream parsing performance + # Bound to stream parsing performance. nc = 0 st = time() for c in Commit.iter_items(self.gitrorepo, self.gitrorepo.head): @@ -89,8 +91,8 @@ def test_commit_serialization(self): rwrepo = self.gitrwrepo make_object = rwrepo.odb.store - # direct serialization - deserialization can be tested afterwards - # serialization is probably limited on IO + # Direct serialization - deserialization can be tested afterwards. + # Serialization is probably limited on IO. hc = rwrepo.commit(rwrepo.head) nc = 5000 diff --git a/test/performance/test_odb.py b/test/performance/test_odb.py index 4208c4181..70934ad6b 100644 --- a/test/performance/test_odb.py +++ b/test/performance/test_odb.py @@ -1,4 +1,5 @@ -"""Performance tests for object store""" +"""Performance tests for object store.""" + import sys from time import time @@ -24,7 +25,7 @@ def test_random_access(self): results[0].append(elapsed) # GET TREES - # walk all trees of all commits + # Walk all trees of all commits. st = time() blobs_per_commit = [] nt = 0 @@ -35,7 +36,7 @@ def test_random_access(self): nt += 1 if item.type == "blob": blobs.append(item) - # direct access for speed + # Direct access for speed. # END while trees are there for walking blobs_per_commit.append(blobs) # END for each commit @@ -75,7 +76,7 @@ def test_random_access(self): results[2].append(elapsed) # END for each repo type - # final results + # Final results. for test_name, a, b in results: print( "%s: %f s vs %f s, pure is %f times slower" % (test_name, a, b, b / a), diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index 25e081578..619126921 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -1,4 +1,5 @@ -"""Performance data streaming performance""" +"""Performance tests for data streaming.""" + import os import subprocess import sys @@ -15,13 +16,13 @@ class TestObjDBPerformance(TestBigRepoR): - large_data_size_bytes = 1000 * 1000 * 10 # some MiB should do it - moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB + large_data_size_bytes = 1000 * 1000 * 10 # Some MiB should do it. + moderate_data_size_bytes = 1000 * 1000 * 1 # Just 1 MiB. @with_rw_repo("HEAD", bare=True) def test_large_data_streaming(self, rwrepo): - # TODO: This part overlaps with the same file in gitdb.test.performance.test_stream - # It should be shared if possible + # TODO: This part overlaps with the same file in gitdb.test.performance.test_stream. + # It should be shared if possible. ldb = LooseObjectDB(osp.join(rwrepo.git_dir, "objects")) for randomize in range(2): @@ -32,7 +33,7 @@ def test_large_data_streaming(self, rwrepo): elapsed = time() - st print("Done (in %f s)" % elapsed, file=sys.stderr) - # writing - due to the compression it will seem faster than it is + # Writing - due to the compression it will seem faster than it is. st = time() binsha = ldb.store(IStream("blob", size, stream)).binsha elapsed_add = time() - st @@ -45,7 +46,7 @@ def test_large_data_streaming(self, rwrepo): msg %= (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) print(msg, file=sys.stderr) - # reading all at once + # Reading all at once. st = time() ostream = ldb.stream(binsha) shadata = ostream.read() @@ -57,7 +58,7 @@ def test_large_data_streaming(self, rwrepo): msg %= (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) print(msg, file=sys.stderr) - # reading in chunks of 1 MiB + # Reading in chunks of 1 MiB. cs = 512 * 1000 chunks = [] st = time() @@ -86,7 +87,7 @@ def test_large_data_streaming(self, rwrepo): file=sys.stderr, ) - # del db file so git has something to do + # del db file so git has something to do. ostream = None import gc @@ -95,11 +96,11 @@ def test_large_data_streaming(self, rwrepo): # VS. CGIT ########## - # CGIT ! Can using the cgit programs be faster ? + # CGIT! Can using the cgit programs be faster? proc = rwrepo.git.hash_object("-w", "--stdin", as_process=True, istream=subprocess.PIPE) - # write file - pump everything in at once to be a fast as possible - data = stream.getvalue() # cache it + # Write file - pump everything in at once to be a fast as possible. + data = stream.getvalue() # Cache it. st = time() proc.stdin.write(data) proc.stdin.close() @@ -107,22 +108,22 @@ def test_large_data_streaming(self, rwrepo): proc.wait() gelapsed_add = time() - st del data - assert gitsha == bin_to_hex(binsha) # we do it the same way, right ? + assert gitsha == bin_to_hex(binsha) # We do it the same way, right? - # as its the same sha, we reuse our path + # As it's the same sha, we reuse our path. fsize_kib = osp.getsize(db_file) / 1000 msg = "Added %i KiB (filesize = %i KiB) of %s data to using git-hash-object in %f s ( %f Write KiB / s)" msg %= (size_kib, fsize_kib, desc, gelapsed_add, size_kib / gelapsed_add) print(msg, file=sys.stderr) - # compare ... + # Compare. print( "Git-Python is %f %% faster than git when adding big %s files" % (100.0 - (elapsed_add / gelapsed_add) * 100, desc), file=sys.stderr, ) - # read all + # Read all. st = time() _hexsha, _typename, size, data = rwrepo.git.get_object_data(gitsha) gelapsed_readall = time() - st @@ -132,14 +133,14 @@ def test_large_data_streaming(self, rwrepo): file=sys.stderr, ) - # compare + # Compare. print( "Git-Python is %f %% faster than git when reading big %sfiles" % (100.0 - (elapsed_readall / gelapsed_readall) * 100, desc), file=sys.stderr, ) - # read chunks + # Read chunks. st = time() _hexsha, _typename, size, stream = rwrepo.git.stream_object_data(gitsha) while True: @@ -158,7 +159,7 @@ def test_large_data_streaming(self, rwrepo): ) print(msg, file=sys.stderr) - # compare + # Compare. print( "Git-Python is %f %% faster than git when reading big %s files in chunks" % (100.0 - (elapsed_readchunks / gelapsed_readchunks) * 100, desc), From 8d3efc54ea3be84b5d29fe388ebe566cc4e8f704 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 01:39:49 -0400 Subject: [PATCH 0822/1790] Remove explicit inheritance from object in test suite --- test/lib/helper.py | 2 +- test/test_index.py | 2 +- test/test_submodule.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 03ae62239..8725cd13f 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -65,7 +65,7 @@ def fixture(name): # { Adapters -class StringProcessAdapter(object): +class StringProcessAdapter: """Allows strings to be used as process objects returned by subprocess.Popen. This is tailored to work with the test system only. diff --git a/test/test_index.py b/test/test_index.py index c73d25543..3c88ce8e1 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -868,7 +868,7 @@ def test_add_a_file_with_wildcard_chars(self, rw_dir): def test__to_relative_path_at_root(self): root = osp.abspath(os.sep) - class Mocked(object): + class Mocked: bare = False git_dir = root working_tree_dir = root diff --git a/test/test_submodule.py b/test/test_submodule.py index e6e54c858..3eec41ee1 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1044,7 +1044,7 @@ def test_branch_renames(self, rw_dir): @skipUnless(is_win, "Specifically for Windows.") def test_to_relative_path_with_super_at_root_drive(self): - class Repo(object): + class Repo: working_tree_dir = "D:\\" super_repo = Repo() From a5fc1d863be74db0e2b9c4e962937e9ee4efb8a4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 01:55:36 -0400 Subject: [PATCH 0823/1790] Improve consistency of "END" comments In the git module (including modules it contains). This does not add any new "# end" or "# END" comments. Instead, it improves the consistency and clarity of existing ones by converting each "# end" comment to "# END" (since capitalization was not used systematically to indicate nesting level or other semantic information, and capitalizing "END" makes clear the nature of the comment), and by adding some information to a few of the comments. This also removes end comments that did not provide information or significant visual indication that a "section" was ending, or where the visual indication they provided was at least as well provided by replacing them with a blank line. However, it does *not* attempt to apply this change everywhere it might be applicable. --- git/cmd.py | 10 ++++---- git/config.py | 28 +++++++++++----------- git/diff.py | 8 +++---- git/index/base.py | 6 ++--- git/index/fun.py | 2 +- git/objects/commit.py | 6 ++--- git/objects/submodule/base.py | 44 ++++++++++++++++------------------- git/objects/submodule/root.py | 4 ++-- git/refs/symbolic.py | 2 +- git/remote.py | 7 +++--- git/repo/base.py | 12 +++++----- git/repo/fun.py | 2 +- git/util.py | 3 ++- 13 files changed, 64 insertions(+), 70 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index de7d34bf5..f76c19a66 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -979,7 +979,7 @@ def execute( else: cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable maybe_patch_caller_env = contextlib.nullcontext() - # end handle + # END handle stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") if shell is None: @@ -1048,7 +1048,7 @@ def kill_process(pid: int) -> None: pass return - # end + # END def kill_process if kill_after_timeout is not None: kill_check = threading.Event() @@ -1100,7 +1100,7 @@ def kill_process(pid: int) -> None: def as_text(stdout_value: Union[bytes, str]) -> str: return not output_stream and safe_decode(stdout_value) or "" - # end + # END def as_text if stderr_value: log.info( @@ -1304,9 +1304,9 @@ def _call_process( "Couldn't find argument '%s' in args %s to insert cmd options after" % (insert_after_this_arg, str(ext_args)) ) from err - # end handle error + # END handle error args_list = ext_args[: index + 1] + opt_args + ext_args[index + 1 :] - # end handle opts_kwargs + # END handle opts_kwargs call = [self.GIT_PYTHON_GIT_EXECUTABLE] diff --git a/git/config.py b/git/config.py index f0002658b..020d823d0 100644 --- a/git/config.py +++ b/git/config.py @@ -446,12 +446,11 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: def string_decode(v: str) -> str: if v[-1] == "\\": v = v[:-1] - # end cut trailing escapes to prevent decode error + # END cut trailing escapes to prevent decode error return v.encode(defenc).decode("unicode_escape") - # end - # end + # END def string_decode while True: # We assume to read binary! @@ -496,12 +495,12 @@ def string_decode(v: str) -> str: optval = optval.strip() if optval == '""': optval = "" - # end handle empty string + # END handle empty string optname = self.optionxform(optname.rstrip()) if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"': is_multi_line = True optval = string_decode(optval[1:]) - # end handle multi-line + # END handle multi-line # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) else: @@ -516,7 +515,7 @@ def string_decode(v: str) -> str: if line.endswith('"'): is_multi_line = False line = line[:-1] - # end handle quotations + # END handle quotations optval = cursect.getlast(optname) cursect.setlast(optname, optval + string_decode(line)) # END parse section or option @@ -600,7 +599,7 @@ def read(self) -> None: # type: ignore[override] files_to_read = [self._file_or_files] else: # for lists or tuples files_to_read = list(self._file_or_files) - # end ensure we have a copy of the paths to handle + # END ensure we have a copy of the paths to handle seen = set(files_to_read) num_read_include_files = 0 @@ -631,11 +630,11 @@ def read(self) -> None: # type: ignore[override] if not osp.isabs(include_path): if not file_ok: continue - # end ignore relative paths if we don't know the configuration file path + # END ignore relative paths if we don't know the configuration file path file_path = cast(PathLike, file_path) assert osp.isabs(file_path), "Need absolute paths to be sure our cycle checks will work" include_path = osp.join(osp.dirname(file_path), include_path) - # end make include path absolute + # END make include path absolute include_path = osp.normpath(include_path) if include_path in seen or not os.access(include_path, os.R_OK): continue @@ -643,15 +642,14 @@ def read(self) -> None: # type: ignore[override] # Insert included file to the top to be considered first. files_to_read.insert(0, include_path) num_read_include_files += 1 - # each include path in configuration file - # end handle includes + # END each include path in configuration file + # END handle includes # END for each file object to read # If there was no file included, we can safely write back (potentially) the # configuration file without altering its meaning. if num_read_include_files == 0: self._merge_includes = False - # end def _write(self, fp: IO) -> None: """Write an .ini-format representation of the configuration state in @@ -714,7 +712,7 @@ def write(self) -> None: "Cannot write back if there is not exactly a single file to write to, have %i files" % len(self._file_or_files) ) - # end assert multiple files + # END assert multiple files if self._has_includes(): log.debug( @@ -722,7 +720,7 @@ def write(self) -> None: + "Set merge_includes=False to prevent this." ) return None - # end + # END stop if we have include files fp = self._file_or_files @@ -904,7 +902,7 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": new_section = self._sections[new_name] for k, vs in self.items_all(section): new_section.setall(k, vs) - # end for each value to copy + # END for each value to copy # This call writes back the changes, which is why we don't have the respective decorator. self.remove_section(section) diff --git a/git/diff.py b/git/diff.py index 829d53924..acf32af7d 100644 --- a/git/diff.py +++ b/git/diff.py @@ -442,7 +442,7 @@ def __str__(self) -> str: msg += self.diff.decode(defenc) if isinstance(self.diff, bytes) else self.diff except UnicodeDecodeError: msg += "OMITTED BINARY DATA" - # end handle encoding + # END handle encoding msg += "\n---" # END diff info @@ -543,7 +543,7 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn # and then retro-actively assign it to our index. if previous_header is not None: index[-1].diff = text[previous_header.end() : _header.start()] - # end assign actual diff + # END assign actual diff # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid. # We just use the one mode we should have parsed. @@ -571,10 +571,10 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn previous_header = _header header = _header - # end for each header we parse + # END for each header we parse if index and header: index[-1].diff = text[header.end() :] - # end assign last diff + # END assign last diff return index diff --git a/git/index/base.py b/git/index/base.py index a43afb320..f7c519df4 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -406,7 +406,7 @@ def raise_exc(e: Exception) -> NoReturn: if S_ISLNK(st.st_mode): yield abs_path.replace(rs, "") continue - # end check symlink + # END check symlink # If the path is not already pointing to an existing file, resolve globs if possible. if not os.path.exists(abs_path) and ("?" in abs_path or "*" in abs_path or "[" in abs_path): @@ -689,7 +689,7 @@ def _entries_for_paths( gitrelative_path = path if self.repo.working_tree_dir: abspath = osp.join(self.repo.working_tree_dir, gitrelative_path) - # end obtain relative and absolute paths + # END obtain relative and absolute paths blob = Blob( self.repo, @@ -867,7 +867,7 @@ def handle_null_entries(self: "IndexFile") -> None: ) # END for each entry index - # end closure + # END closure handle_null_entries(self) # END null_entry handling diff --git a/git/index/fun.py b/git/index/fun.py index ace0866f4..a35990d6d 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -123,7 +123,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: stdout = force_text(stdout, defenc) stderr = force_text(stderr, defenc) raise HookExecutionError(hp, process.returncode, stderr, stdout) - # end handle return code + # END handle return code def stat_mode_to_index_mode(mode: int) -> int: diff --git a/git/objects/commit.py b/git/objects/commit.py index 29123a44f..d39ca52bb 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -553,7 +553,7 @@ def create_from_tree( for p in parent_commits: if not isinstance(p, cls): raise ValueError(f"Parent commit '{p!r}' must be of type {cls}") - # end check parent commit types + # END check parent commit types # END if parent commits are unset # Retrieve all additional information, create a commit object, and serialize it. @@ -730,7 +730,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": next_line = readline() while next_line.startswith(b" "): next_line = readline() - # end skip mergetags + # END skip mergetags # Now we can have the encoding line, or an empty line followed by the optional message. self.encoding = self.default_encoding @@ -754,7 +754,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": is_next_header = True break sig += sigbuf[1:] - # end read all signature + # END read all signature self.gpgsig = sig.rstrip(b"\n").decode(self.encoding, "ignore") if is_next_header: continue diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 24ea5569c..74a6abd54 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -145,7 +145,7 @@ def _set_cache_(self, attr: str) -> None: "This submodule instance does not exist anymore in '%s' file" % osp.join(self.repo.working_tree_dir, ".gitmodules") ) from e - # end + self._url = reader.get("url") # git-python extension values - optional self._branch_path = reader.get_value(self.k_head_option, git.Head.to_full_path(self.k_head_default)) @@ -213,7 +213,7 @@ def _config_parser( except ValueError: # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref. pass - # end handle parent_commit + # END handle parent_commit fp_module: Union[str, BytesIO] if not repo.bare and parent_matches_head and repo.working_tree_dir: fp_module = osp.join(repo.working_tree_dir, cls.k_modules_file) @@ -257,7 +257,7 @@ def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: pc: Union["Commit_ish", None] = self.parent_commit except ValueError: pc = None - # end handle empty parent repository + # END handle empty parent repository parser = self._config_parser(self.repo, pc, read_only) parser.set_submodule(self) return SectionConstraint(parser, sm_section(self.name)) @@ -269,7 +269,6 @@ def _module_abspath(cls, parent_repo: "Repo", path: PathLike, name: str) -> Path if parent_repo.working_tree_dir: return osp.join(parent_repo.working_tree_dir, path) raise NotADirectoryError() - # end @classmethod def _clone_repo( @@ -300,7 +299,6 @@ def _clone_repo( if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) module_checkout_path = osp.join(str(repo.working_tree_dir), path) - # end clone = git.Repo.clone_from( url, @@ -311,7 +309,7 @@ def _clone_repo( ) if cls._need_gitfile_submodules(repo.git): cls._write_git_file_and_module_config(module_checkout_path, module_abspath) - # end + return clone @classmethod @@ -333,8 +331,8 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: path = path[len(working_tree_linux.rstrip("/")) + 1 :] if not path: raise ValueError("Absolute submodule path '%s' didn't yield a valid relative path" % path) - # end verify converted relative path makes sense - # end convert to a relative path + # END verify converted relative path makes sense + # END convert to a relative path return path @@ -792,15 +790,15 @@ def update( ) log.info(msg) may_reset = False - # end handle force - # end handle if we are in the future + # END handle force + # END handle if we are in the future if may_reset and not force and mrepo.is_dirty(index=True, working_tree=True, untracked_files=True): raise RepositoryDirtyError(mrepo, "Cannot reset a dirty repository") - # end handle force and dirty state - # end handle empty repo + # END handle force and dirty state + # END handle empty repo - # end verify future/past + # END verify future/past progress.update( BEGIN | UPDWKTREE, 0, @@ -831,7 +829,7 @@ def update( if not keep_going: raise log.error(str(err)) - # end handle keep_going + # END handle keep_going # HANDLE RECURSION ################## @@ -926,7 +924,7 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = if osp.isfile(osp.join(module_checkout_abspath, ".git")): module_abspath = self._module_abspath(self.repo, self.path, self.name) self._write_git_file_and_module_config(module_checkout_abspath, module_abspath) - # end handle git file rewrite + # END handle git file rewrite # END move physical module # Rename the index entry - we have to manipulate the index directly as @@ -959,7 +957,6 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = # Auto-rename submodule if it's name was 'default', that is, the checkout directory. if previous_sm_path == self.name: self.rename(module_checkout_path) - # end return self @@ -1007,12 +1004,12 @@ def remove( nc += 1 csm.remove(module, force, configuration, dry_run) del csm - # end + if configuration and not dry_run and nc > 0: # Ensure we don't leave the parent repository in a dirty state, and commit our changes. # It's important for recursive, unforced, deletions to work as expected. self.module().index.commit("Removed at least one of child-modules of '%s'" % self.name) - # end handle recursion + # END handle recursion # DELETE REPOSITORY WORKING TREE ################################ @@ -1088,7 +1085,7 @@ def remove( if not dry_run and osp.isdir(git_dir): self._clear_cache() rmtree(git_dir) - # end handle separate bare repository + # END handle separate bare repository # END handle module deletion # Void our data so as not to delay invalid access. @@ -1137,7 +1134,7 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) if commit is None: self._parent_commit = None return self - # end handle None + # END handle None pcommit = self.repo.commit(commit) pctree = pcommit.tree if self.k_modules_file not in pctree: @@ -1162,7 +1159,6 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) self.binsha = pctree[str(self.path)].binsha except KeyError: self.binsha = self.NULL_BIN_SHA - # end self._clear_cache() return self @@ -1236,11 +1232,11 @@ def rename(self, new_name: str) -> "Submodule": tmp_dir = self._module_abspath(self.repo, self.path, str(uuid.uuid4())) os.renames(source_dir, tmp_dir) source_dir = tmp_dir - # end handle self-containment + # END handle self-containment os.renames(source_dir, destination_module_abspath) if mod.working_tree_dir: self._write_git_file_and_module_config(mod.working_tree_dir, destination_module_abspath) - # end move separate git repository + # END move separate git repository return self @@ -1439,7 +1435,7 @@ def iter_items( sm._name = n if pc != repo.commit(): sm._parent_commit = pc - # end set only if not most recent! + # END set only if not most recent! sm._branch_path = git.Head.to_full_path(b) sm._url = u diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index bc61f4ac2..0ac8a22db 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -356,7 +356,7 @@ def update( # to be sure... for remote in smmr: remote.fetch(progress=progress) - # end for each remote + # END for each remote try: tbr = git.Head.create( @@ -391,7 +391,7 @@ def update( if not keep_going: raise log.error(str(err)) - # end handle keep_going + # END handle keep_going # FINALLY UPDATE ALL ACTUAL SUBMODULES ###################################### diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 3587a52d3..7866772e3 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -523,7 +523,7 @@ def log_append( committer_or_reader: Union["Actor", "GitConfigParser"] = self.commit.committer except ValueError: committer_or_reader = self.repo.config_reader() - # end handle newly cloned repositories + # END handle newly cloned repositories if newbinsha is None: newbinsha = self.commit.binsha diff --git a/git/remote.py b/git/remote.py index 76352087a..08101bb6c 100644 --- a/git/remote.py +++ b/git/remote.py @@ -622,7 +622,6 @@ def exists(self) -> bool: return True except cp.NoSectionError: return False - # end @classmethod def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote"]: @@ -757,7 +756,7 @@ def stale_refs(self) -> IterableList[Reference]: else: fqhn = "%s/%s" % (RemoteReference._common_path_default, ref_name) out_refs.append(RemoteReference(self.repo, fqhn)) - # end special case handling + # END special case handling # END for each line return out_refs @@ -887,8 +886,8 @@ def _get_fetch_info_from_stderr( fetch_head_info = fetch_head_info[:l_fil] else: fetch_info_lines = fetch_info_lines[:l_fhi] - # end truncate correct list - # end sanity check + sanitization + # END truncate correct list + # END sanity check + sanitization for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info): try: diff --git a/git/repo/base.py b/git/repo/base.py index 5d0a06198..51c83d720 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -728,7 +728,7 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: """ if len(rev) < 2: raise ValueError("Please specify at least two revs, got only %i" % len(rev)) - # end handle input + # END handle input res: List[Union[Commit_ish, None]] = [] try: @@ -736,15 +736,15 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: except GitCommandError as err: if err.status == 128: raise - # end handle invalid rev + # END handle invalid rev # Status code 1 is returned if there is no merge-base # (see https://github.com/git/git/blob/master/builtin/merge-base.c#L16) return res - # end exception handling + # END exception handling for line in lines: res.append(self.commit(line)) - # end for each merge-base + # END for each merge-base return res @@ -1098,7 +1098,7 @@ class InfoTD(TypedDict, total=False): parts = self.re_whitespace.split(line_str, 1) firstpart = parts[0] is_binary = False - # end handle decode of line + # END handle decode of line if self.re_hexsha_only.search(firstpart): # handles @@ -1448,7 +1448,7 @@ def archive( path = cast(Union[PathLike, List[PathLike], Tuple[PathLike, ...]], path) if not isinstance(path, (tuple, list)): path = [path] - # end ensure paths is list (or tuple) + # END ensure paths is list (or tuple) self.git.archive("--", treeish, *path, **kwargs) return self diff --git a/git/repo/fun.py b/git/repo/fun.py index f0bb4cd1f..29a899ea8 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -118,7 +118,7 @@ def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: if not osp.isabs(path): path = osp.normpath(osp.join(osp.dirname(d), path)) return find_submodule_git_dir(path) - # end handle exception + # END handle exception return None diff --git a/git/util.py b/git/util.py index adbc8146f..78d6169ac 100644 --- a/git/util.py +++ b/git/util.py @@ -668,7 +668,8 @@ def new_message_handler(self) -> Callable[[str], None]: def handler(line: AnyStr) -> None: return self._parse_progress_line(line.rstrip()) - # end + # END def handler + return handler def line_dropped(self, line: str) -> None: From c2eb6b544f1f65deb493e49d9a0cabdf8a714966 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 02:02:38 -0400 Subject: [PATCH 0824/1790] Add missing comment revisions in git/objects/submodule/base.py --- git/objects/submodule/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 74a6abd54..41abe9223 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -136,7 +136,7 @@ def __init__( def _set_cache_(self, attr: str) -> None: if attr in ("path", "_url", "_branch_path"): reader: SectionConstraint = self.config_reader() - # default submodule values + # Default submodule values. try: self.path = reader.get("path") except cp.NoSectionError as e: @@ -147,7 +147,7 @@ def _set_cache_(self, attr: str) -> None: ) from e self._url = reader.get("url") - # git-python extension values - optional + # GitPython extension values - optional. self._branch_path = reader.get_value(self.k_head_option, git.Head.to_full_path(self.k_head_default)) elif attr == "_name": raise AttributeError("Cannot retrieve the name of a submodule if it was not set initially") @@ -484,7 +484,7 @@ def add( # END verify we have url url = urls[0] else: - # clone new repo + # Clone new repo. kwargs: Dict[str, Union[bool, int, str, Sequence[TBD]]] = {"n": no_checkout} if not branch_is_default: kwargs["b"] = br.name @@ -531,7 +531,7 @@ def add( sm._url = url if not branch_is_default: - # store full path + # Store full path. writer.set_value(cls.k_head_option, br.path) sm._branch_path = br.path From ddb44178788f63e207641ab579bf7031b0f5c8dc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 02:07:05 -0400 Subject: [PATCH 0825/1790] Improve consistency of "END" comments in test suite As in the recent roughly corresponding change inside git/, this does not add new END comments (and actually removes some), but instead focuses on making the existing ones more consistent in style, mainly adjusting capitalization (turning "end" to "END"). --- test/performance/lib.py | 2 +- test/test_base.py | 2 +- test/test_config.py | 6 +----- test/test_diff.py | 2 +- test/test_docs.py | 2 -- test/test_index.py | 2 +- test/test_remote.py | 3 +-- test/test_repo.py | 8 ++++---- test/test_submodule.py | 15 +++++++-------- 9 files changed, 17 insertions(+), 25 deletions(-) diff --git a/test/performance/lib.py b/test/performance/lib.py index 8c68db782..8e76fe815 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -45,7 +45,7 @@ def setUp(self): k_env_git_repo, ) repo_path = osp.dirname(__file__) - # end set some repo path + # END set some repo path self.gitrorepo = Repo(repo_path, odbt=GitCmdObjectDB, search_parent_directories=True) self.puregitrorepo = Repo(repo_path, odbt=GitDB, search_parent_directories=True) diff --git a/test/test_base.py b/test/test_base.py index 94a268ecd..051b96354 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -142,5 +142,5 @@ def test_add_unicode(self, rw_repo): else: # On POSIX, we can just add Unicode files without problems. rw_repo.git.add(rw_repo.working_dir) - # end + rw_repo.index.commit("message") diff --git a/test/test_config.py b/test/test_config.py index 2e278bec2..63fbc61e6 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -190,13 +190,9 @@ def test_config_include(self, rw_dir): def write_test_value(cw, value): cw.set_value(value, "value", value) - # end - def check_test_value(cr, value): assert cr.get_value(value, "value") == value - # end - # PREPARE CONFIG FILE A fpa = osp.join(rw_dir, "a") with GitConfigParser(fpa, read_only=False) as cw: @@ -225,7 +221,7 @@ def check_test_value(cr, value): with GitConfigParser(fpa, read_only=True) as cr: for tv in ("a", "b", "c"): check_test_value(cr, tv) - # end for each test to verify + # END for each test to verify assert len(cr.items("include")) == 8, "Expected all include sections to be merged" # Test writable config writers - assure write-back doesn't involve includes. diff --git a/test/test_diff.py b/test/test_diff.py index 1d9c398c2..a46555ce7 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -223,7 +223,7 @@ def test_diff_index(self): for dr in res: self.assertTrue(dr.diff.startswith(b"@@"), dr) self.assertIsNotNone(str(dr), "Diff to string conversion should be possible") - # end for each diff + # END for each diff dr = res[3] assert dr.diff.endswith(b"+Binary files a/rps and b/rps differ\n") diff --git a/test/test_docs.py b/test/test_docs.py index 1ca07f886..206560edd 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -146,8 +146,6 @@ def update(self, op_code, cur_count, max_count=None, message=""): message or "NO MESSAGE", ) - # end - self.assertEqual(len(cloned_repo.remotes), 1) # We have been cloned, so should be one remote. self.assertEqual(len(bare_repo.remotes), 0) # This one was just initialized. origin = bare_repo.create_remote("origin", url=cloned_repo.working_tree_dir) diff --git a/test/test_index.py b/test/test_index.py index 3c88ce8e1..95da472e7 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -635,7 +635,7 @@ def mixed_iterator(): ) os.remove(link_file) - # end for each target + # END for each target # END real symlink test # Add fake symlink and assure it checks-our as symlink. diff --git a/test/test_remote.py b/test/test_remote.py index b102730ac..3e7a025c5 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -518,8 +518,7 @@ def test_base(self, rw_repo, remote_repo): if branch.name != "master": branch.delete(remote_repo, branch, force=True) num_deleted += 1 - # end - # end for each branch + # END for each branch self.assertGreater(num_deleted, 0) self.assertEqual( len(rw_repo.remotes.origin.fetch(prune=True)), diff --git a/test/test_repo.py b/test/test_repo.py index c7c13dc7f..a9d52ced0 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -790,7 +790,7 @@ def test_untracked_files(self, rwrepo): repo_add(untracked_files) self.assertEqual(len(rwrepo.untracked_files), (num_recently_untracked - len(files))) - # end for each run + # END for each run def test_config_reader(self): reader = self.rorepo.config_reader() # All config files. @@ -1165,8 +1165,8 @@ def last_commit(repo, rev, path): for repo_type in (GitCmdObjectDB, GitDB): repo = Repo(self.rorepo.working_tree_dir, odbt=repo_type) last_commit(repo, "master", "test/test_base.py") - # end for each repository type - # end for each iteration + # END for each repository type + # END for each iteration def test_remote_method(self): self.assertRaises(ValueError, self.rorepo.remote, "foo-blue") @@ -1227,7 +1227,7 @@ def test_merge_base(self): res = repo.merge_base(c1, c2, c3, **{kw: True}) self.assertIsInstance(res, list) self.assertEqual(len(res), 1) - # end for each keyword signalling all merge-bases to be returned + # END for each keyword signalling all merge-bases to be returned # Test for no merge base - can't do as we have. self.assertRaises(GitCommandError, repo.merge_base, c1, "ffffff") diff --git a/test/test_submodule.py b/test/test_submodule.py index 3eec41ee1..da918b4d0 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -704,7 +704,7 @@ def test_first_submodule(self, rwrepo): sm = rwrepo.create_submodule(sm_name, sm_path, rwrepo.git_dir, no_checkout=True) assert sm.exists() and sm.module_exists() rwrepo.index.commit("Added submodule " + sm_name) - # end for each submodule path to add + # END for each submodule path to add self.assertRaises(ValueError, rwrepo.create_submodule, "fail", osp.expanduser("~")) self.assertRaises( @@ -731,7 +731,7 @@ def test_add_empty_repo(self, rwdir): url=empty_repo_dir, no_checkout=checkout_mode and True or False, ) - # end for each checkout mode + # END for each checkout mode @with_rw_directory @_patch_git_config("protocol.file.allow", "always") @@ -783,8 +783,8 @@ def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): for init in (False, True): sm.update(init=init) sm2.update(init=init) - # end for each init state - # end for each iteration + # END for each init state + # END for each iteration sm.move(sm.path + "_moved") sm2.move(sm2.path + "_moved") @@ -839,7 +839,7 @@ def assert_exists(sm, value=True): assert sm.exists() == value assert sm.module_exists() == value - # end + # END def assert_exists # As git is backwards compatible itself, it would still recognize what we do here... unless we really # muss it up. That's the only reason why the test is still here... @@ -854,7 +854,7 @@ def assert_exists(sm, value=True): assert osp.isfile(module_repo_path) assert sm.module().has_separate_working_tree() assert find_submodule_git_dir(module_repo_path) is not None, "module pointed to by .git file must be valid" - # end verify submodule 'style' + # END verify submodule 'style' # Test move. new_sm_path = join_path_native("submodules", "one") @@ -911,7 +911,7 @@ def assert_exists(sm, value=True): sm.remove(dry_run=dry_run, force=True) assert_exists(sm, value=dry_run) assert osp.isdir(sm_module_path) == dry_run - # end for each dry-run mode + # END for each dry-run mode @with_rw_directory def test_ignore_non_submodule_file(self, rwdir): @@ -974,7 +974,6 @@ def test_rename(self, rwdir): sm_mod = sm.module() if osp.isfile(osp.join(sm_mod.working_tree_dir, ".git")) == sm._need_gitfile_submodules(parent.git): assert sm_mod.git_dir.endswith(join_path_native(".git", "modules", new_sm_name)) - # end @with_rw_directory def test_branch_renames(self, rw_dir): From 1d3f275841a96ea4ba40b3813114258cc886b9fc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 05:27:11 -0400 Subject: [PATCH 0826/1790] Avoid making "END" notation more verbose This changes the style "END" comments appearing after local function definitions from "END def X" to "END X". Note that this still does not add such comments where they were not present, it only adjusts their style where they already existed. --- git/cmd.py | 4 ++-- git/config.py | 2 +- git/index/base.py | 1 + git/util.py | 2 +- test/test_submodule.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index f76c19a66..5359924f0 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1048,7 +1048,7 @@ def kill_process(pid: int) -> None: pass return - # END def kill_process + # END kill_process if kill_after_timeout is not None: kill_check = threading.Event() @@ -1100,7 +1100,7 @@ def kill_process(pid: int) -> None: def as_text(stdout_value: Union[bytes, str]) -> str: return not output_stream and safe_decode(stdout_value) or "" - # END def as_text + # END as_text if stderr_value: log.info( diff --git a/git/config.py b/git/config.py index 020d823d0..8f68c4e66 100644 --- a/git/config.py +++ b/git/config.py @@ -450,7 +450,7 @@ def string_decode(v: str) -> str: return v.encode(defenc).decode("unicode_escape") - # END def string_decode + # END string_decode while True: # We assume to read binary! diff --git a/git/index/base.py b/git/index/base.py index f7c519df4..be6137a4f 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -868,6 +868,7 @@ def handle_null_entries(self: "IndexFile") -> None: # END for each entry index # END closure + handle_null_entries(self) # END null_entry handling diff --git a/git/util.py b/git/util.py index 78d6169ac..2c3d6f33f 100644 --- a/git/util.py +++ b/git/util.py @@ -668,7 +668,7 @@ def new_message_handler(self) -> Callable[[str], None]: def handler(line: AnyStr) -> None: return self._parse_progress_line(line.rstrip()) - # END def handler + # END handler return handler diff --git a/test/test_submodule.py b/test/test_submodule.py index da918b4d0..6b331b3bc 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -839,7 +839,7 @@ def assert_exists(sm, value=True): assert sm.exists() == value assert sm.module_exists() == value - # END def assert_exists + # END assert_exists # As git is backwards compatible itself, it would still recognize what we do here... unless we really # muss it up. That's the only reason why the test is still here... From 81dad7e62230501444620f0a5fd479a4cd9eda19 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 05:33:58 -0400 Subject: [PATCH 0827/1790] Add missing comment revisions in git/index/base.py --- git/index/base.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index be6137a4f..a101fd371 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1023,8 +1023,8 @@ def move( was_dry_run = kwargs.pop("dry_run", kwargs.pop("n", None)) kwargs["dry_run"] = True - # First execute rename in dryrun so the command tells us what it actually does. - # (for later output) + # First execute rename in dryrun so the command tells us what it actually does + # (for later output). out = [] mvlines = self.repo.git.mv(args, paths, **kwargs).splitlines() @@ -1186,7 +1186,7 @@ def checkout( def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLike]) -> None: stderr_IO = proc.stderr if not stderr_IO: - return None # return early if stderr empty + return None # Return early if stderr empty. else: stderr_bytes = stderr_IO.read() # line contents: @@ -1252,9 +1252,9 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik if isinstance(paths, str): paths = [paths] - # make sure we have our entries loaded before we start checkout_index + # Make sure we have our entries loaded before we start checkout_index, # which will hold a lock on it. We try to get the lock as well during - # our entries initialization + # our entries initialization. self.entries args.append("--stdin") @@ -1267,7 +1267,7 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik for path in paths: co_path = to_native_path_linux(self._to_relative_path(path)) - # if the item is not in the index, it could be a directory + # If the item is not in the index, it could be a directory. path_is_directory = False try: @@ -1347,8 +1347,8 @@ def reset( If you want git-reset like behaviour, use *HEAD.reset* instead. :return: self""" - # what we actually want to do is to merge the tree into our existing - # index, which is what git-read-tree does + # What we actually want to do is to merge the tree into our existing + # index, which is what git-read-tree does. new_inst = type(self).from_tree(self.repo, commit) if not paths: self.entries = new_inst.entries @@ -1360,7 +1360,7 @@ def reset( key = entry_key(path, 0) self.entries[key] = nie[key] except KeyError: - # if key is not in theirs, it musn't be in ours + # If key is not in theirs, it musn't be in ours. try: del self.entries[key] except KeyError: @@ -1391,7 +1391,7 @@ def diff( ) -> git_diff.DiffIndex: """Diff this index against the working copy or a Tree or Commit object. - For a documentation of the parameters and return values, see + For documentation of the parameters and return values, see :meth:`Diffable.diff `. :note: @@ -1413,7 +1413,7 @@ def diff( other = self.repo.rev_parse(other) # END object conversion - if isinstance(other, Object): # for Tree or Commit + if isinstance(other, Object): # For Tree or Commit. # Invert the existing R flag. cur_val = kwargs.get("R", False) kwargs["R"] = not cur_val From a040edbf09a5fd2d2355c7042d313edcad74eb40 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 07:40:01 -0400 Subject: [PATCH 0828/1790] Shorten some docstring references with tilde notation This uses the Sphinx ~ notation to express references like: :class:`Z ` In the abbreviated form, which Sphinx renders the same way: :class:`~a.b.c.Z` (This commit also makes some other polishing changes to affected and nearby text.) --- git/diff.py | 20 +++++++++----------- git/objects/base.py | 2 +- git/objects/submodule/base.py | 18 ++++++++++-------- git/refs/head.py | 4 ++-- git/refs/reference.py | 4 ++-- git/refs/symbolic.py | 32 +++++++++++++++++--------------- git/refs/tag.py | 4 ++-- git/remote.py | 20 ++++++++++---------- git/repo/base.py | 35 +++++++++++++++++------------------ 9 files changed, 70 insertions(+), 69 deletions(-) diff --git a/git/diff.py b/git/diff.py index acf32af7d..275534bbf 100644 --- a/git/diff.py +++ b/git/diff.py @@ -118,12 +118,12 @@ def diff( This the item to compare us with. * If None, we will be compared to the working tree. - * If :class:`Treeish `, it will be compared against - the respective tree. - * If :class:`Index `, it will be compared against the index. + * If :class:`~git.index.base.Treeish`, it will be compared against the + respective tree. + * If :class:`~Diffable.Index`, it will be compared against the index. * If :attr:`git.NULL_TREE`, it will compare against the empty tree. - * It defaults to :class:`Index ` so that the method will not - by default fail on bare repositories. + * It defaults to :class:`~Diffable.Index` so that the method will not by + default fail on bare repositories. :param paths: This a list of paths or a single path to limit the diff to. It will only @@ -141,11 +141,9 @@ def diff( :return: git.DiffIndex :note: - On a bare repository, 'other' needs to be provided - as :class:`Index `, - or as :class:`Tree ` - or :class:`Commit `, or a git command error will - occur. + On a bare repository, 'other' needs to be provided as + :class:`~Diffable.Index`, or as :class:`~git.objects.tree.Tree` or + :class:`~git.objects.commit.Commit`, or a git command error will occur. """ args: List[Union[PathLike, Diffable, Type["Diffable.Index"], object]] = [] args.append("--abbrev=40") # We need full shas. @@ -222,7 +220,7 @@ class DiffIndex(List[T_Diff]): def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: """ :return: - iterator yielding :class:`Diff` instances that match the given `change_type` + Iterator yielding :class:`Diff` instances that match the given `change_type` :param change_type: Member of :attr:`DiffIndex.change_type`, namely: diff --git a/git/objects/base.py b/git/objects/base.py index 0d88aa185..a771f9fbf 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -178,7 +178,7 @@ def __init__( ) -> None: """Initialize a newly instanced IndexObject. - :param repo: The :class:`Repo ` we are located in. + :param repo: The :class:`~git.repo.base.Repo` we are located in. :param binsha: 20 byte sha1. :param mode: The stat compatible file mode as int, use the :mod:`stat` module to evaluate diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 41abe9223..018f9a39c 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -113,14 +113,16 @@ def __init__( ) -> None: """Initialize this instance with its attributes. - We only document the parameters that differ - from :class:`IndexObject `. - - :param repo: Our parent repository - :param binsha: binary sha referring to a commit in the remote repository, see url parameter - :param parent_commit: see set_parent_commit() - :param url: The url to the remote repository which is the submodule - :param branch_path: full (relative) path to ref to checkout when cloning the remote repository + We only document the parameters that differ from + :class:`~git.objects.base.IndexObject`. + + :param repo: Our parent repository. + :param binsha: Binary sha referring to a commit in the remote repository. See + the ``url`` parameter. + :param parent_commit: See :meth:`set_parent_commit`. + :param url: The URL to the remote repository which is the submodule. + :param branch_path: Full (relative) path to ref to checkout when cloning the + remote repository. """ super(Submodule, self).__init__(repo, binsha, mode, path) self.size = 0 diff --git a/git/refs/head.py b/git/refs/head.py index 194f51e78..30ff336d5 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -231,8 +231,8 @@ def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: :param force: If True, changes to the index and the working tree will be discarded. - If False, :class:`GitCommandError ` will be - raised in that situation. + If False, :class:`~git.exc.GitCommandError` will be raised in that + situation. :param kwargs: Additional keyword arguments to be passed to git checkout, e.g. diff --git a/git/refs/reference.py b/git/refs/reference.py index ca265c0e4..8a0262ea4 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -40,8 +40,8 @@ def wrapper(self: T_References, *args: Any) -> _T: class Reference(SymbolicReference, LazyMixin, IterableObj): """A named reference to any object. - Subclasses may apply restrictions though, e.g., a :class:`Head ` - can only point to commits. + Subclasses may apply restrictions though, e.g., a :class:`~git.refs.head.Head` can + only point to commits. """ __slots__ = () diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 7866772e3..6e4b657d6 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -59,7 +59,7 @@ class SymbolicReference: """Special case of a reference that is symbolic. This does not point to a specific commit, but to another - :class:`Head `, which itself specifies a commit. + :class:`~git.refs.head.Head`, which itself specifies a commit. A typical example for a symbolic reference is ``HEAD``. """ @@ -345,7 +345,7 @@ def set_object( If the reference does not exist, it will be created. :param object: A refspec, a :class:`SymbolicReference` or an - :class:`Object ` instance. + :class:`~git.objects.base.Object` instance. :class:`SymbolicReference` instances will be dereferenced beforehand to obtain the object they point to. :param logmsg: If not None, the message will be used in the reflog entry to be @@ -396,10 +396,10 @@ def set_reference( symbolic one. :param ref: - A :class:`SymbolicReference` instance, - an :class:`Object ` instance, or a refspec string. - Only if the ref is a :class:`SymbolicReference` instance, we will point to - it. Everything else is dereferenced to obtain the actual object. + A :class:`SymbolicReference` instance, an :class:`~git.objects.base.Object` + instance, or a refspec string. Only if the ref is a + :class:`SymbolicReference` instance, we will point to it. Everything else is + dereferenced to obtain the actual object. :param logmsg: If set to a string, the message will be used in the reflog. Otherwise, a reflog entry is not written for the changed reference. @@ -514,7 +514,7 @@ def log_append( :param message: A message describing the change. :param newbinsha: The sha the ref points to now. If None, our current commit sha will be used. - :return: The added :class:`RefLogEntry ` instance. + :return: The added :class:`~git.refs.log.RefLogEntry` instance. """ # NOTE: We use the committer of the currently active commit - this should be # correct to allow overriding the committer on a per-commit level. @@ -540,7 +540,7 @@ def log_entry(self, index: int) -> "RefLogEntry": .. note:: This method must read part of the reflog during execution, hence it should be used sparingly, or only if you need just one index. - In that case, it will be faster than the ``log()`` method. + In that case, it will be faster than the :meth:`log` method. """ return RefLog.entry_at(RefLog.path(self), index) @@ -816,8 +816,8 @@ def iter_items( which is not detached and pointing to a valid ref. The list is lexicographically sorted. The returned objects are instances of - concrete subclasses, such as :class:`Head ` or - :class:`TagReference `. + concrete subclasses, such as :class:`~git.refs.head.Head` or + :class:`~git.refs.tag.TagReference`. """ return (r for r in cls._iter_items(repo, common_path) if r.__class__ is SymbolicReference or not r.is_detached) @@ -826,14 +826,16 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere """ Make a symbolic reference from a path. - :param path: Full ``.git``-directory-relative path name to the Reference to instantiate. + :param path: Full ``.git``-directory-relative path name to the Reference to + instantiate. - :note: Use :meth:`to_full_path` if you only have a partial path of a known Reference type. + :note: Use :meth:`to_full_path` if you only have a partial path of a known + Reference type. :return: - Instance of type :class:`Reference `, - :class:`Head `, or :class:`Tag `, - depending on the given path. + Instance of type :class:`~git.refs.reference.Reference`, + :class:`~git.refs.head.Head`, or :class:`~git.refs.tag.Tag`, depending on + the given path. """ if not path: raise ValueError("Cannot create Reference from %r" % path) diff --git a/git/refs/tag.py b/git/refs/tag.py index 3c269d9ba..d00adc121 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -89,8 +89,8 @@ def create( The prefix ``refs/tags`` is implied. :param ref: - A reference to the :class:`Object ` you want to - tag. The Object can be a commit, tree or blob. + A reference to the :class:`~git.objects.base.Object` you want to tag. + The Object can be a commit, tree or blob. :param logmsg: If not None, the message will be used in your tag object. This will also diff --git a/git/remote.py b/git/remote.py index 08101bb6c..880e87232 100644 --- a/git/remote.py +++ b/git/remote.py @@ -210,9 +210,9 @@ def old_commit(self) -> Union[str, SymbolicReference, Commit_ish, None]: def remote_ref(self) -> Union[RemoteReference, TagReference]: """ :return: - Remote :class:`Reference ` or - :class:`TagReference ` in the local repository - corresponding to the :attr:`remote_ref_string` kept in this instance. + Remote :class:`~git.refs.reference.Reference` or + :class:`~git.refs.tag.TagReference` in the local repository corresponding to + the :attr:`remote_ref_string` kept in this instance. """ # Translate heads to a local remote. Tags stay as they are. if self.remote_ref_string.startswith("refs/tags"): @@ -1093,7 +1093,7 @@ def push( `Click here `__ for a description of all arguments given to the function. * An instance of a class derived from :class:`git.RemoteProgress` that - overrides the :meth:`update ` method. + overrides the :meth:`~git.RemoteProgress.update` method. :note: No further progress information is returned after push returns. @@ -1109,14 +1109,14 @@ def push( :param kwargs: Additional arguments to be passed to git-push. :return: - A :class:`PushInfoList` object, where each list member - represents an individual head which had been updated on the remote side. + A :class:`PushInfoList` object, where each list member represents an + individual head which had been updated on the remote side. If the push contains rejected heads, these will have the :attr:`PushInfo.ERROR` bit set in their flags. - If the operation fails completely, the length of the returned PushInfoList will - be 0. - Call :meth:`raise_if_error ` on the returned - object to raise on any failure. + If the operation fails completely, the length of the returned PushInfoList + will be 0. + Call :meth:`~PushInfoList.raise_if_error` on the returned object to raise on + any failure. """ kwargs = add_progress(kwargs, self.repo.git, progress) diff --git a/git/repo/base.py b/git/repo/base.py index 51c83d720..411a321ee 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -387,7 +387,7 @@ def heads(self) -> "IterableList[Head]": def references(self) -> "IterableList[Reference]": """A list of Reference objects representing tags, heads and remote references. - :return: IterableList(Reference, ...) + :return: ``git.IterableList(Reference, ...)`` """ return Reference.list_items(self) @@ -400,11 +400,11 @@ def references(self) -> "IterableList[Reference]": @property def index(self) -> "IndexFile": """ - :return: :class:`IndexFile ` representing this - repository's index. + :return: A :class:`~git.index.base.IndexFile` representing this repository's + index. :note: This property can be expensive, as the returned - :class:`IndexFile ` will be reinitialized. + :class:`~git.index.base.IndexFile` will be reinitialized. It is recommended to reuse the object. """ return IndexFile(self) @@ -520,7 +520,7 @@ def create_head( :note: For more documentation, please see the :meth:`Head.create ` method. - :return: Newly created :class:`Head ` Reference + :return: Newly created :class:`~git.refs.head.Head` Reference """ return Head.create(self, path, commit, logmsg, force) @@ -544,7 +544,7 @@ def create_tag( :note: For more documentation, please see the :meth:`TagReference.create ` method. - :return: :class:`TagReference ` object + :return: :class:`~git.refs.tag.TagReference` object """ return TagReference.create(self, path, ref, message, force, **kwargs) @@ -558,7 +558,7 @@ def create_remote(self, name: str, url: str, **kwargs: Any) -> Remote: For more information, please see the documentation of the :meth:`Remote.create ` method. - :return: :class:`Remote ` reference + :return: :class:`~git.remote.Remote` reference """ return Remote.create(self, name, url, **kwargs) @@ -599,8 +599,8 @@ def config_reader( ) -> GitConfigParser: """ :return: - :class:`GitConfigParser ` allowing to read the - full git configuration, but not to write it. + :class:`~git.config.GitConfigParser` allowing to read the full git + configuration, but not to write it. The configuration will include values from the system, user and repository configuration files. @@ -633,11 +633,10 @@ def _config_reader( def config_writer(self, config_level: Lit_config_levels = "repository") -> GitConfigParser: """ :return: - A :class:`GitConfigParser ` allowing to write - values of the specified configuration file level. Config writers should be - retrieved, used to change the configuration, and written right away as they - will lock the configuration file in question and prevent other's to write - it. + A :class:`~git.config.GitConfigParser` allowing to write values of the + specified configuration file level. Config writers should be retrieved, used + to change the configuration, and written right away as they will lock the + configuration file in question and prevent other's to write it. :param config_level: One of the following values: @@ -720,10 +719,10 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: :param rev: At least two revs to find the common ancestor for. :param kwargs: Additional arguments to be passed to the ``repo.git.merge_base()`` command which does all the work. - :return: A list of :class:`Commit ` objects. If - ``--all`` was not passed as a keyword argument, the list will have at max - one :class:`Commit `, or is empty if no common - merge base exists. + :return: A list of :class:`~git.objects.commit.Commit` objects. If ``--all`` was + not passed as a keyword argument, the list will have at max one + :class:`~git.objects.commit.Commit`, or is empty if no common merge base + exists. :raises ValueError: If not at least two revs are provided. """ if len(rev) < 2: From af1b5d4a2e7dac065c19129b448bf76866a4afa2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 13:39:56 -0400 Subject: [PATCH 0829/1790] Fix commented *.py names at the top of modules A number of them were incorrect, due to how files were renamed, and code moved between them, over time. Some were corrected earlier, and most of the rest are corrected here. Not all modules have these at all. This does not add any to modules that don't have them. It only fixes them where present but wrong. --- git/index/base.py | 2 +- git/objects/tag.py | 2 +- git/repo/base.py | 2 +- git/util.py | 2 +- test/test_db.py | 2 +- test/test_docs.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index a101fd371..b87fefd56 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1,4 +1,4 @@ -# index.py +# base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under diff --git a/git/objects/tag.py b/git/objects/tag.py index 55f7e19da..a7ff7f263 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -1,4 +1,4 @@ -# objects.py +# tag.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under diff --git a/git/repo/base.py b/git/repo/base.py index 411a321ee..4790ea4e7 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1,4 +1,4 @@ -# repo.py +# base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under diff --git a/git/util.py b/git/util.py index 2c3d6f33f..73b3a2edd 100644 --- a/git/util.py +++ b/git/util.py @@ -1,4 +1,4 @@ -# utils.py +# util.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under diff --git a/test/test_db.py b/test/test_db.py index acf8379db..d59aa6cc0 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,4 +1,4 @@ -# test_repo.py +# test_db.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under diff --git a/test/test_docs.py b/test/test_docs.py index 206560edd..73f85d540 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# test_git.py +# test_docs.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under From b970d425754df37e9f70d55544c8fdf2e6eb7b29 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 14:29:20 -0400 Subject: [PATCH 0830/1790] Remove encoding declarations In Python 3, UTF-8 is the default encoding when no encoding declaration is present, and omitting the encoding declaration when UTF-8 is intended is officially suggested. - https://docs.python.org/3/reference/lexical_analysis.html - https://pep8.org/#source-file-encoding Furthermore, not all files in this project had them (and their presence or absence seems to have been mainly for historical reasons rather than file contents). --- doc/source/conf.py | 4 +--- git/compat.py | 1 - git/types.py | 1 - test/test_base.py | 1 - test/test_clone.py | 1 - test/test_commit.py | 1 - test/test_diff.py | 1 - test/test_docs.py | 1 - test/test_exc.py | 1 - test/test_git.py | 1 - test/test_index.py | 1 - test/test_repo.py | 1 - test/test_submodule.py | 1 - 13 files changed, 1 insertion(+), 15 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index d19b0c0ec..9c22ca06a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# GitPython documentation build configuration file, created by +# GitPython documentation build configuration file, originally created by # sphinx-quickstart on Sat Jan 24 11:51:01 2009. # # This file is execfile()d with the current directory set to its containing dir. diff --git a/git/compat.py b/git/compat.py index 93e06d2ec..f17e52f7b 100644 --- a/git/compat.py +++ b/git/compat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # compat.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/git/types.py b/git/types.py index 22bb91c16..2709bbf34 100644 --- a/git/types.py +++ b/git/types.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/test_base.py b/test/test_base.py index 051b96354..e4704c7d8 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_clone.py b/test/test_clone.py index f66130cf0..7624b317b 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/test_commit.py b/test/test_commit.py index 3f3c52f1b..21d17ae2c 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_commit.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_diff.py b/test/test_diff.py index a46555ce7..50b96efff 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -1,4 +1,3 @@ -# coding: utf-8 # test_diff.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_docs.py b/test/test_docs.py index 73f85d540..394b58b5f 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_docs.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_exc.py b/test/test_exc.py index 0c3ff35a8..ad43695b3 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_exc.py # Copyright (C) 2008, 2009, 2016 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_git.py b/test/test_git.py index aa7a415e5..814316edd 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_git.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_index.py b/test/test_index.py index 95da472e7..bf582bad9 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_index.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_repo.py b/test/test_repo.py index a9d52ced0..1ba85acf9 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_repo.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # diff --git a/test/test_submodule.py b/test/test_submodule.py index 6b331b3bc..f63db1495 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # This module is part of GitPython and is released under # the BSD License: https://opensource.org/license/bsd-3-clause/ From 36a3b7618479f200e0d4ad0fca38dab217db7dc5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Nov 2023 12:40:24 -0400 Subject: [PATCH 0831/1790] Revise some comments and strings These are a few things I had missed in #1725. --- git/config.py | 1 + git/objects/base.py | 4 ++-- git/refs/symbolic.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/git/config.py b/git/config.py index 8f68c4e66..1931f142d 100644 --- a/git/config.py +++ b/git/config.py @@ -150,6 +150,7 @@ class SectionConstraint(Generic[T_ConfigParser]): """ __slots__ = ("_config", "_section_name") + _valid_attrs_ = ( "get_value", "set_value", diff --git a/git/objects/base.py b/git/objects/base.py index a771f9fbf..6d2efa32a 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -137,7 +137,7 @@ def __repr__(self) -> str: @property def hexsha(self) -> str: """:return: 40 byte hex version of our 20 byte binary sha""" - # b2a_hex produces bytes + # b2a_hex produces bytes. return bin_to_hex(self.binsha).decode("ascii") @property @@ -206,7 +206,7 @@ def __hash__(self) -> int: def _set_cache_(self, attr: str) -> None: if attr in IndexObject.__slots__: - # they cannot be retrieved lateron ( not without searching for them ) + # They cannot be retrieved later on (not without searching for them). raise AttributeError( "Attribute '%s' unset: path and mode attributes must have been set during %s object creation" % (attr, type(self).__name__) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 6e4b657d6..73f64a57b 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -215,7 +215,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: elif any(component.endswith(".lock") for component in str(ref_path).split("/")): raise ValueError( f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" - f" '.lock'" + " '.lock'" ) @classmethod @@ -309,8 +309,8 @@ def set_commit( ) -> "SymbolicReference": """As set_object, but restricts the type of object to be a Commit. - :raise ValueError: If commit is not a Commit object or doesn't point to - a commit + :raise ValueError: If commit is not a :class:`~git.objects.commit.Commit` object + or doesn't point to a commit :return: self """ # Check the type - assume the best if it is a base-string. From a47e46d7512c595dc230b2caeedb4ba615fadfa9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Nov 2023 13:30:12 -0400 Subject: [PATCH 0832/1790] Use zero-argument super() where applicable This replaces 2-argument super(...) calls with zero-argument super() calls, where doing so preserves the exact semantics. This turns out to be everywhere in the actual code (now all uses of the super builtin are calls to it with zero arguments), and also in most places code was discussed in comments. The single exception, occurring in comments (technically a string literal, but being used as a comment), appears in git.objects.util.TraversableIterableObj, where super(Commit, self), rather than super(TraversableIterableObj, self) which would have the same meaning as super(), is shown. It may be that this should be changed, but such a revision would not signify identical semantics, and may require other changes to preserve or recover clarity or documentary accuracy. --- git/cmd.py | 4 ++-- git/config.py | 30 +++++++++++++++--------------- git/db.py | 2 +- git/exc.py | 6 +++--- git/index/base.py | 4 ++-- git/objects/base.py | 8 ++++---- git/objects/commit.py | 4 ++-- git/objects/submodule/base.py | 6 +++--- git/objects/submodule/root.py | 2 +- git/objects/submodule/util.py | 4 ++-- git/objects/tag.py | 4 ++-- git/objects/tree.py | 10 +++++----- git/objects/util.py | 6 ++---- git/refs/head.py | 2 +- git/refs/log.py | 2 +- git/refs/reference.py | 4 ++-- git/refs/remote.py | 2 +- git/remote.py | 6 +++--- git/util.py | 8 ++++---- test/performance/lib.py | 6 +++--- test/test_commit.py | 2 +- test/test_git.py | 2 +- test/test_index.py | 2 +- test/test_remote.py | 4 ++-- 24 files changed, 64 insertions(+), 66 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 5359924f0..fd39a8eeb 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -718,7 +718,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None): It is meant to be the working tree directory if available, or the ``.git`` directory in case of bare repositories. """ - super(Git, self).__init__() + super().__init__() self._working_dir = expand_path(working_dir) self._git_options: Union[List[str], Tuple[str, ...]] = () self._persistent_git_options: List[str] = [] @@ -765,7 +765,7 @@ def _set_cache_(self, attr: str) -> None: tuple(int(n) for n in version_numbers.split(".")[:4] if n.isdigit()), ) else: - super(Git, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle version info @property diff --git a/git/config.py b/git/config.py index 1931f142d..2cb057021 100644 --- a/git/config.py +++ b/git/config.py @@ -107,7 +107,7 @@ def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParse # END for each base # END if mutating methods configuration is set - new_type = super(MetaParserBuilder, cls).__new__(cls, name, bases, clsdict) + new_type = super().__new__(cls, name, bases, clsdict) return new_type @@ -178,7 +178,7 @@ def __del__(self) -> None: def __getattr__(self, attr: str) -> Any: if attr in self._valid_attrs_: return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs) - return super(SectionConstraint, self).__getattribute__(attr) + return super().__getattribute__(attr) def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any: """Call the configuration at the given method which must take a section name @@ -207,36 +207,36 @@ class _OMD(OrderedDict_OMD): """Ordered multi-dict.""" def __setitem__(self, key: str, value: _T) -> None: - super(_OMD, self).__setitem__(key, [value]) + super().__setitem__(key, [value]) def add(self, key: str, value: Any) -> None: if key not in self: - super(_OMD, self).__setitem__(key, [value]) + super().__setitem__(key, [value]) return None - super(_OMD, self).__getitem__(key).append(value) + super().__getitem__(key).append(value) def setall(self, key: str, values: List[_T]) -> None: - super(_OMD, self).__setitem__(key, values) + super().__setitem__(key, values) def __getitem__(self, key: str) -> Any: - return super(_OMD, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] def getlast(self, key: str) -> Any: - return super(_OMD, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] def setlast(self, key: str, value: Any) -> None: if key not in self: - super(_OMD, self).__setitem__(key, [value]) + super().__setitem__(key, [value]) return - prior = super(_OMD, self).__getitem__(key) + prior = super().__getitem__(key) prior[-1] = value def get(self, key: str, default: Union[_T, None] = None) -> Union[_T, None]: - return super(_OMD, self).get(key, [default])[-1] + return super().get(key, [default])[-1] def getall(self, key: str) -> List[_T]: - return super(_OMD, self).__getitem__(key) + return super().__getitem__(key) def items(self) -> List[Tuple[str, _T]]: # type: ignore[override] """List of (key, last value for key).""" @@ -680,7 +680,7 @@ def write_section(name: str, section_dict: _OMD) -> None: def items(self, section_name: str) -> List[Tuple[str, str]]: # type: ignore[override] """:return: list((option, value), ...) pairs of all items in the given section""" - return [(k, v) for k, v in super(GitConfigParser, self).items(section_name) if k != "__name__"] + return [(k, v) for k, v in super().items(section_name) if k != "__name__"] def items_all(self, section_name: str) -> List[Tuple[str, List[str]]]: """:return: list((option, [values...]), ...) pairs of all items in the given section""" @@ -748,7 +748,7 @@ def _assure_writable(self, method_name: str) -> None: def add_section(self, section: str) -> None: """Assures added options will stay in order""" - return super(GitConfigParser, self).add_section(section) + return super().add_section(section) @property def read_only(self) -> bool: @@ -899,7 +899,7 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": if self.has_section(new_name): raise ValueError("Destination section '%s' already exists" % new_name) - super(GitConfigParser, self).add_section(new_name) + super().add_section(new_name) new_section = self._sections[new_name] for k, vs in self.items_all(section): new_section.setall(k, vs) diff --git a/git/db.py b/git/db.py index 1aacd0c84..9e278ea75 100644 --- a/git/db.py +++ b/git/db.py @@ -34,7 +34,7 @@ class GitCmdObjectDB(LooseObjectDB): def __init__(self, root_path: PathLike, git: "Git") -> None: """Initialize this instance with the root and a git command.""" - super(GitCmdObjectDB, self).__init__(root_path) + super().__init__(root_path) self._git = git def info(self, binsha: bytes) -> OInfo: diff --git a/git/exc.py b/git/exc.py index bfb023fa5..124c5eeea 100644 --- a/git/exc.py +++ b/git/exc.py @@ -137,7 +137,7 @@ class GitCommandNotFound(CommandError): the GIT_PYTHON_GIT_EXECUTABLE environment variable.""" def __init__(self, command: Union[List[str], Tuple[str], str], cause: Union[str, Exception]) -> None: - super(GitCommandNotFound, self).__init__(command, cause) + super().__init__(command, cause) self._msg = "Cmd('%s') not found%s" @@ -151,7 +151,7 @@ def __init__( stderr: Union[bytes, str, None] = None, stdout: Union[bytes, str, None] = None, ) -> None: - super(GitCommandError, self).__init__(command, status, stderr, stdout) + super().__init__(command, status, stderr, stdout) class CheckoutError(GitError): @@ -207,7 +207,7 @@ def __init__( stderr: Union[bytes, str, None] = None, stdout: Union[bytes, str, None] = None, ) -> None: - super(HookExecutionError, self).__init__(command, status, stderr, stdout) + super().__init__(command, status, stderr, stdout) self._msg = "Hook('%s') failed%s" diff --git a/git/index/base.py b/git/index/base.py index b87fefd56..c2333a2c2 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -153,7 +153,7 @@ def _set_cache_(self, attr: str) -> None: self._deserialize(stream) else: - super(IndexFile, self)._set_cache_(attr) + super()._set_cache_(attr) def _index_path(self) -> PathLike: if self.repo.git_dir: @@ -1425,4 +1425,4 @@ def diff( raise ValueError("other must be None, Diffable.Index, a Tree or Commit, was %r" % other) # Diff against working copy - can be handled by superclass natively. - return super(IndexFile, self).diff(other, paths, create_patch, **kwargs) + return super().diff(other, paths, create_patch, **kwargs) diff --git a/git/objects/base.py b/git/objects/base.py index 6d2efa32a..9f188a955 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -62,7 +62,7 @@ def __init__(self, repo: "Repo", binsha: bytes): :param binsha: 20 byte SHA1 """ - super(Object, self).__init__() + super().__init__() self.repo = repo self.binsha = binsha assert len(binsha) == 20, "Require 20 byte binary sha, got %r, len = %i" % ( @@ -108,7 +108,7 @@ def _set_cache_(self, attr: str) -> None: self.size = oinfo.size # type: int # assert oinfo.type == self.type, _assertion_msg_format % (self.binsha, oinfo.type, self.type) else: - super(Object, self)._set_cache_(attr) + super()._set_cache_(attr) def __eq__(self, other: Any) -> bool: """:return: True if the objects have the same SHA1""" @@ -190,7 +190,7 @@ def __init__( Path may not be set if the index object has been created directly, as it cannot be retrieved without knowing the parent tree. """ - super(IndexObject, self).__init__(repo, binsha) + super().__init__(repo, binsha) if mode is not None: self.mode = mode if path is not None: @@ -212,7 +212,7 @@ def _set_cache_(self, attr: str) -> None: % (attr, type(self).__name__) ) else: - super(IndexObject, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle slot attribute @property diff --git a/git/objects/commit.py b/git/objects/commit.py index d39ca52bb..04acb668b 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -146,7 +146,7 @@ def __init__( as what time.altzone returns. The sign is inverted compared to git's UTC timezone. """ - super(Commit, self).__init__(repo, binsha) + super().__init__(repo, binsha) self.binsha = binsha if tree is not None: assert isinstance(tree, Tree), "Tree needs to be a Tree instance, was %s" % type(tree) @@ -218,7 +218,7 @@ def _set_cache_(self, attr: str) -> None: _binsha, _typename, self.size, stream = self.repo.odb.stream(self.binsha) self._deserialize(BytesIO(stream.read())) else: - super(Commit, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs @property diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 018f9a39c..1516306ec 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -124,7 +124,7 @@ def __init__( :param branch_path: Full (relative) path to ref to checkout when cloning the remote repository. """ - super(Submodule, self).__init__(repo, binsha, mode, path) + super().__init__(repo, binsha, mode, path) self.size = 0 self._parent_commit = parent_commit if url is not None: @@ -154,7 +154,7 @@ def _set_cache_(self, attr: str) -> None: elif attr == "_name": raise AttributeError("Cannot retrieve the name of a submodule if it was not set initially") else: - super(Submodule, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attribute name @classmethod @@ -174,7 +174,7 @@ def __eq__(self, other: Any) -> bool: """Compare with another submodule.""" # We may only compare by name as this should be the ID they are hashed with. # Otherwise this type wouldn't be hashable. - # return self.path == other.path and self.url == other.url and super(Submodule, self).__eq__(other) + # return self.path == other.path and self.url == other.url and super().__eq__(other) return self._name == other._name def __ne__(self, other: object) -> bool: diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 0ac8a22db..cfcbb4cb7 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -55,7 +55,7 @@ class RootModule(Submodule): def __init__(self, repo: "Repo"): # repo, binsha, mode=None, path=None, name = None, parent_commit=None, url=None, ref=None) - super(RootModule, self).__init__( + super().__init__( repo, binsha=self.NULL_BIN_SHA, mode=self.k_default_mode, diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index e13528a8f..3fc0b0b56 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -79,7 +79,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._smref: Union["ReferenceType[Submodule]", None] = None self._index = None self._auto_write = True - super(SubmoduleConfigParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # { Interface def set_submodule(self, submodule: "Submodule") -> None: @@ -107,7 +107,7 @@ def flush_to_index(self) -> None: # { Overridden Methods def write(self) -> None: # type: ignore[override] - rval: None = super(SubmoduleConfigParser, self).write() + rval: None = super().write() self.flush_to_index() return rval diff --git a/git/objects/tag.py b/git/objects/tag.py index a7ff7f263..6eb1c8d90 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -64,7 +64,7 @@ def __init__( The timezone that the authored_date is in, in a format similar to :attr:`time.altzone`. """ - super(TagObject, self).__init__(repo, binsha) + super().__init__(repo, binsha) if object is not None: self.object: Union["Commit", "Blob", "Tree", "TagObject"] = object if tag is not None: @@ -108,4 +108,4 @@ def _set_cache_(self, attr: str) -> None: self.message = "" # END check our attributes else: - super(TagObject, self)._set_cache_(attr) + super()._set_cache_(attr) diff --git a/git/objects/tree.py b/git/objects/tree.py index 708ab3edd..1be6f193e 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -237,7 +237,7 @@ def __init__( mode: int = tree_id << 12, path: Union[PathLike, None] = None, ): - super(Tree, self).__init__(repo, binsha, mode, path) + super().__init__(repo, binsha, mode, path) @classmethod def _get_intermediate_items( @@ -254,7 +254,7 @@ def _set_cache_(self, attr: str) -> None: ostream = self.repo.odb.stream(self.binsha) self._cache: List[TreeCacheTup] = tree_entries_from_data(ostream.read()) else: - super(Tree, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attribute def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[IndexObjUnion]: @@ -352,13 +352,13 @@ def traverse( # def is_tree_traversed(inp: Tuple) -> TypeGuard[Tuple[Iterator[Union['Tree', 'Blob', 'Submodule']]]]: # return all(isinstance(x, (Blob, Tree, Submodule)) for x in inp[1]) - # ret = super(Tree, self).traverse(predicate, prune, depth, branch_first, visit_once, ignore_self) + # ret = super().traverse(predicate, prune, depth, branch_first, visit_once, ignore_self) # ret_tup = itertools.tee(ret, 2) # assert is_tree_traversed(ret_tup), f"Type is {[type(x) for x in list(ret_tup[0])]}" # return ret_tup[0]""" return cast( Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]], - super(Tree, self)._traverse( + super()._traverse( predicate, prune, depth, # type: ignore @@ -374,7 +374,7 @@ def list_traverse(self, *args: Any, **kwargs: Any) -> IterableList[IndexObjUnion traverse() Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] """ - return super(Tree, self)._list_traverse(*args, **kwargs) + return super()._list_traverse(*args, **kwargs) # List protocol diff --git a/git/objects/util.py b/git/objects/util.py index 5ccb3ec37..7af7fa0e5 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -577,7 +577,7 @@ class TraversableIterableObj(IterableObj, Traversable): TIobj_tuple = Tuple[Union[T_TIobj, None], T_TIobj] def list_traverse(self: T_TIobj, *args: Any, **kwargs: Any) -> IterableList[T_TIobj]: - return super(TraversableIterableObj, self)._list_traverse(*args, **kwargs) + return super()._list_traverse(*args, **kwargs) @overload # type: ignore def traverse(self: T_TIobj) -> Iterator[T_TIobj]: @@ -652,7 +652,5 @@ def is_commit_traversed(inp: Tuple) -> TypeGuard[Tuple[Iterator[Tuple['Commit', """ return cast( Union[Iterator[T_TIobj], Iterator[Tuple[Union[None, T_TIobj], T_TIobj]]], - super(TraversableIterableObj, self)._traverse( - predicate, prune, depth, branch_first, visit_once, ignore_self, as_edge # type: ignore - ), + super()._traverse(predicate, prune, depth, branch_first, visit_once, ignore_self, as_edge), # type: ignore ) diff --git a/git/refs/head.py b/git/refs/head.py index 30ff336d5..fa40943c6 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -39,7 +39,7 @@ class HEAD(SymbolicReference): def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) - super(HEAD, self).__init__(repo, path) + super().__init__(repo, path) self.commit: "Commit" def orig_head(self) -> SymbolicReference: diff --git a/git/refs/log.py b/git/refs/log.py index 21c757ccd..ebdaf04d1 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -155,7 +155,7 @@ class RefLog(List[RefLogEntry], Serializable): __slots__ = ("_path",) def __new__(cls, filepath: Union[PathLike, None] = None) -> "RefLog": - inst = super(RefLog, cls).__new__(cls) + inst = super().__new__(cls) return inst def __init__(self, filepath: Union[PathLike, None] = None): diff --git a/git/refs/reference.py b/git/refs/reference.py index 8a0262ea4..f0eb6bfaa 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -62,7 +62,7 @@ def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> Non if check_path and not str(path).startswith(self._common_path_default + "/"): raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}") self.path: str # SymbolicReference converts to string at the moment. - super(Reference, self).__init__(repo, path) + super().__init__(repo, path) def __str__(self) -> str: return self.name @@ -87,7 +87,7 @@ def set_object( # END handle commit retrieval # END handle message is set - super(Reference, self).set_object(object, logmsg) + super().set_object(object, logmsg) if oldbinsha is not None: # From refs/files-backend.c in git-source: diff --git a/git/refs/remote.py b/git/refs/remote.py index e4b1f4392..f26ee08fc 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -40,7 +40,7 @@ def iter_items( common_path = join_path(common_path, str(remote)) # END handle remote constraint # super is Reference - return super(RemoteReference, cls).iter_items(repo, common_path) + return super().iter_items(repo, common_path) # The Head implementation of delete also accepts strs, but this # implementation does not. mypy doesn't have a way of representing diff --git a/git/remote.py b/git/remote.py index 880e87232..ccf70a25c 100644 --- a/git/remote.py +++ b/git/remote.py @@ -573,14 +573,14 @@ def __getattr__(self, attr: str) -> Any: """Allows to call this instance like remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name.""" if attr == "_config_reader": - return super(Remote, self).__getattr__(attr) + return super().__getattr__(attr) # Sometimes, probably due to a bug in Python itself, we are being called # even though a slot of the same name exists. try: return self._config_reader.get(attr) except cp.NoOptionError: - return super(Remote, self).__getattr__(attr) + return super().__getattr__(attr) # END handle exception def _config_section_name(self) -> str: @@ -592,7 +592,7 @@ def _set_cache_(self, attr: str) -> None: # values implicitly, such as in print(r.pushurl). self._config_reader = SectionConstraint(self.repo.config_reader("repository"), self._config_section_name()) else: - super(Remote, self)._set_cache_(attr) + super()._set_cache_(attr) def __str__(self) -> str: return self.name diff --git a/git/util.py b/git/util.py index 73b3a2edd..bd1fbe247 100644 --- a/git/util.py +++ b/git/util.py @@ -718,7 +718,7 @@ class CallableRemoteProgress(RemoteProgress): def __init__(self, fn: Callable) -> None: self._callable = fn - super(CallableRemoteProgress, self).__init__() + super().__init__() def update(self, *args: Any, **kwargs: Any) -> None: self._callable(*args, **kwargs) @@ -1040,7 +1040,7 @@ def __init__( :param max_block_time_s: Maximum amount of seconds we may lock. """ - super(BlockingLockFile, self).__init__(file_path) + super().__init__(file_path) self._check_interval = check_interval_s self._max_block_time = max_block_time_s @@ -1054,7 +1054,7 @@ def _obtain_lock(self) -> None: maxtime = starttime + float(self._max_block_time) while True: try: - super(BlockingLockFile, self)._obtain_lock() + super()._obtain_lock() except IOError as e: # synity check: if the directory leading to the lockfile is not # readable anymore, raise an exception @@ -1103,7 +1103,7 @@ class IterableList(List[T_IterableObj]): __slots__ = ("_id_attr", "_prefix") def __new__(cls, id_attr: str, prefix: str = "") -> "IterableList[T_IterableObj]": - return super(IterableList, cls).__new__(cls) + return super().__new__(cls) def __init__(self, id_attr: str, prefix: str = "") -> None: self._id_attr = id_attr diff --git a/test/performance/lib.py b/test/performance/lib.py index 8e76fe815..1ac3e0b60 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -33,7 +33,7 @@ class TestBigRepoR(TestBase): def setUp(self): try: - super(TestBigRepoR, self).setUp() + super().setUp() except AttributeError: pass @@ -65,7 +65,7 @@ class TestBigRepoRW(TestBigRepoR): def setUp(self): self.gitrwrepo = None try: - super(TestBigRepoRW, self).setUp() + super().setUp() except AttributeError: pass dirname = tempfile.mktemp() @@ -74,7 +74,7 @@ def setUp(self): self.puregitrwrepo = Repo(dirname, odbt=GitDB) def tearDown(self): - super(TestBigRepoRW, self).tearDown() + super().tearDown() if self.gitrwrepo is not None: rmtree(self.gitrwrepo.working_dir) self.gitrwrepo.git.clear_cache() diff --git a/test/test_commit.py b/test/test_commit.py index 21d17ae2c..1327616ed 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -275,7 +275,7 @@ def test_iteration(self): class Child(Commit): def __init__(self, *args, **kwargs): - super(Child, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) child_commits = list(Child.iter_items(self.rorepo, "master", paths=("CHANGES", "AUTHORS"))) assert type(child_commits[0]) is Child diff --git a/test/test_git.py b/test/test_git.py index 814316edd..06f8f5c97 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -32,7 +32,7 @@ class TestGit(TestBase): @classmethod def setUpClass(cls): - super(TestGit, cls).setUpClass() + super().setUpClass() cls.git = Git(cls.rorepo.working_dir) def tearDown(self): diff --git a/test/test_index.py b/test/test_index.py index bf582bad9..3357dc880 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -66,7 +66,7 @@ def _make_hook(git_dir, name, content, make_exec=True): class TestIndex(TestBase): def __init__(self, *args): - super(TestIndex, self).__init__(*args) + super().__init__(*args) self._reset_progress() def _assert_fprogress(self, entries): diff --git a/test/test_remote.py b/test/test_remote.py index 3e7a025c5..8205c0bcd 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -44,7 +44,7 @@ class TestRemoteProgress(RemoteProgress): __slots__ = ("_seen_lines", "_stages_per_op", "_num_progress_messages") def __init__(self): - super(TestRemoteProgress, self).__init__() + super().__init__() self._seen_lines = [] self._stages_per_op = {} self._num_progress_messages = 0 @@ -53,7 +53,7 @@ def _parse_progress_line(self, line): # We may remove the line later if it is dropped. # Keep it for debugging. self._seen_lines.append(line) - rval = super(TestRemoteProgress, self)._parse_progress_line(line) + rval = super()._parse_progress_line(line) return rval def line_dropped(self, line): From 91131770eb0e932b9bc0918fe8374875ecbbd816 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Nov 2023 13:45:32 -0400 Subject: [PATCH 0833/1790] Don't swallow AttributeError from super().setUp() This is conceptually independent of the immediately preceding super() call refactoring, but serves the same goal of simplifying and clarifying super() calls. In test.performance.lib, the TestBigRepoR and TestBigRepoRW classes' setUp methods had calls to setUp through super proxies, which were wrapped in try-blocks to swallow AttributeError exceptions. This removes that, relying on the presence of setUp methods in some parent or sibling class in the MRO. The intent appeared to be solely to account for the possibility that no class in the MRO would define a setUp method. However, the unittest.TestCase base class defines noop setUp and tearDown methods to ensure this does not have to be done. This may also make the code more robust, because the form in which AttributeError was being swallowed was: try: super().setUp() except AttributeError: pass But that has the disadvantage of also catching AttributeError due to a bug or other problem in code that runs *in* an ancestor or sibling class's existing setUp method. This could alternatively be addressed by using: try: other_setUp = super().setUp except AttributeError: pass else: other_setUp() However, because unittest.TestCase provides empty setUp and tearDown methods to allow such special-casing to be avoided (both in cases like this and for the test runner), this isn't needed. --- test/performance/lib.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/performance/lib.py b/test/performance/lib.py index 1ac3e0b60..2b2a632d9 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -32,10 +32,7 @@ class TestBigRepoR(TestBase): """ def setUp(self): - try: - super().setUp() - except AttributeError: - pass + super().setUp() repo_path = os.environ.get(k_env_git_repo) if repo_path is None: @@ -64,10 +61,7 @@ class TestBigRepoRW(TestBigRepoR): def setUp(self): self.gitrwrepo = None - try: - super().setUp() - except AttributeError: - pass + super().setUp() dirname = tempfile.mktemp() os.mkdir(dirname) self.gitrwrepo = self.gitrorepo.clone(dirname, shared=True, bare=True, odbt=GitCmdObjectDB) From 2c7fce0c289d871111ef1610a76785cc8c536cb6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 3 Nov 2023 08:01:13 -0400 Subject: [PATCH 0834/1790] Remove obsolete note in _iter_packed_refs This removes a comment noting that a try-finally block had been present (or had been intended), but was removed because some version of Python had imposed a limitation on yield appearing in try-finally. That comment was obsolete as of 58c5b99 (#326), which wrapped the relevant code in a with-statement, because: 1. Since then, the cleanup is done in a manner equivaent to try-finally. 2. It turned out, as noted in that PR, that cleanup had not always been done automatically. (This was contrary to the prediction given in the comment.) 3. At some point before that, the limitation that had prevented the use of try-finally no longer affected any supported version of Python. Specifically, it appears the only limitation that this could be was the limitation lifted in Python 2.5, where along with the introduction of close(), which is automatically called when a generator object is finalized, it became permitted for yield to appear in a try-block with an associated finally-block, on the grounds that calling close() runs the finally-block (by raising GeneratorExit). For details, see: https://docs.python.org/3/whatsnew/2.5.html#pep-342-new-generator-features (This obsolete comment was one of the things I discovered while working on #1725, but I didn't include this change there, having not yet looked into the history of the code enough to be sure.) --- git/refs/symbolic.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 73f64a57b..99a60201f 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -149,10 +149,6 @@ def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: except OSError: return None # END no packed-refs file handling - # NOTE: Had try-finally block around here to close the fp, - # but some python version wouldn't allow yields within that. - # I believe files are closing themselves on destruction, so it is - # alright. @classmethod def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> str: From 09826f4ecf9cacc617627ad54452235d449345fc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 04:50:02 -0400 Subject: [PATCH 0835/1790] Revise test_util comment style --- test/test_util.py | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index f75231c98..5cac56f3b 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,4 +1,4 @@ -# test_utils.py +# test_util.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under @@ -275,14 +275,14 @@ def test_lock_file(self): my_file = tempfile.mktemp() lock_file = LockFile(my_file) assert not lock_file._has_lock() - # release lock we don't have - fine + # Release lock we don't have - fine. lock_file._release_lock() - # get lock + # Get lock. lock_file._obtain_lock_or_raise() assert lock_file._has_lock() - # concurrent access + # Concurrent access. other_lock_file = LockFile(my_file) assert not other_lock_file._has_lock() self.assertRaises(IOError, other_lock_file._obtain_lock_or_raise) @@ -293,7 +293,7 @@ def test_lock_file(self): other_lock_file._obtain_lock_or_raise() self.assertRaises(IOError, lock_file._obtain_lock_or_raise) - # auto-release on destruction + # Auto-release on destruction. del other_lock_file lock_file._obtain_lock_or_raise() lock_file._release_lock() @@ -303,7 +303,7 @@ def test_blocking_lock_file(self): lock_file = BlockingLockFile(my_file) lock_file._obtain_lock() - # next one waits for the lock + # Next one waits for the lock. start = time.time() wait_time = 0.1 wait_lock = BlockingLockFile(my_file, 0.05, wait_time) @@ -318,7 +318,7 @@ def test_user_id(self): self.assertIn("@", get_user_id()) def test_parse_date(self): - # parse_date(from_timestamp()) must return the tuple unchanged + # parse_date(from_timestamp()) must return the tuple unchanged. for timestamp, offset in ( (1522827734, -7200), (1522827734, 0), @@ -326,7 +326,7 @@ def test_parse_date(self): ): self.assertEqual(parse_date(from_timestamp(timestamp, offset)), (timestamp, offset)) - # test all supported formats + # Test all supported formats. def assert_rval(rval, veri_time, offset=0): self.assertEqual(len(rval), 2) self.assertIsInstance(rval[0], int) @@ -334,7 +334,7 @@ def assert_rval(rval, veri_time, offset=0): self.assertEqual(rval[0], veri_time) self.assertEqual(rval[1], offset) - # now that we are here, test our conversion functions as well + # Now that we are here, test our conversion functions as well. utctz = altz_to_utctz_str(offset) self.assertIsInstance(utctz, str) self.assertEqual(utctz_to_altz(verify_utctz(utctz)), offset) @@ -347,13 +347,13 @@ def assert_rval(rval, veri_time, offset=0): iso3 = ("2005.04.07 22:13:11 -0000", 0) alt = ("04/07/2005 22:13:11", 0) alt2 = ("07.04.2005 22:13:11", 0) - veri_time_utc = 1112911991 # the time this represents, in time since epoch, UTC + veri_time_utc = 1112911991 # The time this represents, in time since epoch, UTC. for date, offset in (rfc, iso, iso2, iso3, alt, alt2): assert_rval(parse_date(date), veri_time_utc, offset) # END for each date type - # and failure - self.assertRaises(ValueError, parse_date, datetime.now()) # non-aware datetime + # ...and failure. + self.assertRaises(ValueError, parse_date, datetime.now()) # Non-aware datetime. self.assertRaises(ValueError, parse_date, "invalid format") self.assertRaises(ValueError, parse_date, "123456789 -02000") self.assertRaises(ValueError, parse_date, " 123456789 -0200") @@ -362,7 +362,7 @@ def test_actor(self): for cr in (None, self.rorepo.config_reader()): self.assertIsInstance(Actor.committer(cr), Actor) self.assertIsInstance(Actor.author(cr), Actor) - # END assure config reader is handled + # END ensure config reader is handled @with_rw_repo("HEAD") @mock.patch("getpass.getuser") @@ -402,7 +402,7 @@ def test_actor_get_uid_laziness_called(self, mock_get_uid): mock_get_uid.return_value = "user" committer = Actor.committer(None) author = Actor.author(None) - # We can't test with `self.rorepo.config_reader()` here, as the uuid laziness + # We can't test with `self.rorepo.config_reader()` here, as the UUID laziness # depends on whether the user running the test has their global user.name config set. self.assertEqual(committer.name, "user") self.assertTrue(committer.email.startswith("user@")) @@ -436,30 +436,30 @@ def test_iterable_list(self, case): self.assertEqual(len(ilist), 2) - # contains works with name and identity + # Contains works with name and identity. self.assertIn(name1, ilist) self.assertIn(name2, ilist) self.assertIn(m2, ilist) self.assertIn(m2, ilist) self.assertNotIn("invalid", ilist) - # with string index + # With string index. self.assertIs(ilist[name1], m1) self.assertIs(ilist[name2], m2) - # with int index + # With int index. self.assertIs(ilist[0], m1) self.assertIs(ilist[1], m2) - # with getattr + # With getattr. self.assertIs(ilist.one, m1) self.assertIs(ilist.two, m2) - # test exceptions + # Test exceptions. self.assertRaises(AttributeError, getattr, ilist, "something") self.assertRaises(IndexError, ilist.__getitem__, "something") - # delete by name and index + # Delete by name and index. self.assertRaises(IndexError, ilist.__delitem__, "something") del ilist[name2] self.assertEqual(len(ilist), 1) @@ -494,21 +494,21 @@ def test_altz_to_utctz_str(self): self.assertEqual(altz_to_utctz_str(-59), "+0000") def test_from_timestamp(self): - # Correct offset: UTC+2, should return datetime + tzoffset(+2) + # Correct offset: UTC+2, should return datetime + tzoffset(+2). altz = utctz_to_altz("+0200") self.assertEqual( datetime.fromtimestamp(1522827734, tzoffset(altz)), from_timestamp(1522827734, altz), ) - # Wrong offset: UTC+58, should return datetime + tzoffset(UTC) + # Wrong offset: UTC+58, should return datetime + tzoffset(UTC). altz = utctz_to_altz("+5800") self.assertEqual( datetime.fromtimestamp(1522827734, tzoffset(0)), from_timestamp(1522827734, altz), ) - # Wrong offset: UTC-9000, should return datetime + tzoffset(UTC) + # Wrong offset: UTC-9000, should return datetime + tzoffset(UTC). altz = utctz_to_altz("-9000") self.assertEqual( datetime.fromtimestamp(1522827734, tzoffset(0)), @@ -538,7 +538,7 @@ def test_remove_password_from_command_line(self): redacted_cmd_1 = remove_password_if_present(cmd_1) assert username not in " ".join(redacted_cmd_1) assert password not in " ".join(redacted_cmd_1) - # Check that we use a copy + # Check that we use a copy. assert cmd_1 is not redacted_cmd_1 assert username in " ".join(cmd_1) assert password in " ".join(cmd_1) From e2fa5e2225fbcdd0f34108f1b9a6b4b1e8653555 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 09:06:11 -0400 Subject: [PATCH 0836/1790] Pull rmtree tests out of TestUtils class And rework them as pure pytest tests (using pytest fixtures). This creates the TestRmtree class, which does not derive from TestCase. --- pyproject.toml | 13 +-- test-requirements.txt | 3 +- test/test_util.py | 203 ++++++++++++++++++++++-------------------- 3 files changed, 115 insertions(+), 104 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f4fc33fec..eae5943ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,10 +3,11 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -python_files = 'test_*.py' -testpaths = 'test' # space separated list of paths from root e.g test tests doc/testing -addopts = '--cov=git --cov-report=term --disable-warnings' -filterwarnings = 'ignore::DeprecationWarning' +addopts = "--cov=git --cov-report=term --disable-warnings" +filterwarnings = "ignore::DeprecationWarning" +python_files = "test_*.py" +tmp_path_retention_policy = "failed" +testpaths = "test" # Space separated list of paths from root e.g test tests doc/testing. # --cov coverage # --cov-report term # send report to terminal term-missing -> terminal with line numbers html xml # --cov-report term-missing # to terminal with line numbers @@ -29,7 +30,7 @@ show_error_codes = true implicit_reexport = true # strict = true -# TODO: remove when 'gitdb' is fully annotated +# TODO: Remove when 'gitdb' is fully annotated. exclude = ["^git/ext/gitdb"] [[tool.mypy.overrides]] module = "gitdb.*" @@ -44,5 +45,5 @@ omit = ["*/git/ext/*"] [tool.black] line-length = 120 -target-version = ['py37'] +target-version = ["py37"] extend-exclude = "git/ext/gitdb" diff --git a/test-requirements.txt b/test-requirements.txt index a69181be1..8cadfb836 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,8 +4,9 @@ ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" mypy pre-commit -pytest +pytest >= 7.3.1 pytest-cov pytest-instafail +pytest-mock pytest-subtests pytest-sugar diff --git a/test/test_util.py b/test/test_util.py index 5cac56f3b..784dc9bfc 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -5,7 +5,6 @@ # the BSD License: https://opensource.org/license/bsd-3-clause/ import ast -import contextlib from datetime import datetime import os import pathlib @@ -15,7 +14,7 @@ import sys import tempfile import time -from unittest import SkipTest, mock, skipIf, skipUnless +from unittest import SkipTest, mock, skipUnless import ddt import pytest @@ -44,123 +43,133 @@ from test.lib import TestBase, with_rw_repo -class _Member: - """A member of an IterableList.""" - - __slots__ = ("name",) - - def __init__(self, name): - self.name = name - - def __repr__(self): - return f"{type(self).__name__}({self.name!r})" - - -@contextlib.contextmanager -def _tmpdir_to_force_permission_error(): - """Context manager to test permission errors in situations where they are not overcome.""" +@pytest.fixture +def permission_error_tmpdir(tmp_path): + """Fixture to test permissions errors situations where they are not overcome.""" if sys.platform == "cygwin": raise SkipTest("Cygwin can't set the permissions that make the test meaningful.") if sys.version_info < (3, 8): raise SkipTest("In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") - with tempfile.TemporaryDirectory() as parent: - td = pathlib.Path(parent, "testdir") - td.mkdir() - (td / "x").write_bytes(b"") - (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. - td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. - yield td + td = tmp_path / "testdir" + td.mkdir() + (td / "x").write_bytes(b"") + (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. + td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. + yield td -@contextlib.contextmanager -def _tmpdir_for_file_not_found(): - """Context manager to test errors deleting a directory that are not due to permissions.""" - with tempfile.TemporaryDirectory() as parent: - yield pathlib.Path(parent, "testdir") # It is deliberately never created. +@pytest.fixture +def file_not_found_tmpdir(tmp_path): + """Fixture to test errors deleting a directory that are not due to permissions.""" + yield tmp_path / "testdir" # It is deliberately never created. -@ddt.ddt -class TestUtils(TestBase): - def test_rmtree_deletes_nested_dir_with_files(self): - with tempfile.TemporaryDirectory() as parent: - td = pathlib.Path(parent, "testdir") - for d in td, td / "q", td / "s": - d.mkdir() - for f in ( - td / "p", - td / "q" / "w", - td / "q" / "x", - td / "r", - td / "s" / "y", - td / "s" / "z", - ): - f.write_bytes(b"") +class TestRmtree: + """Tests for :func:`git.util.rmtree`.""" - try: - rmtree(td) - except SkipTest as ex: - self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + def test_deletes_nested_dir_with_files(self, tmp_path): + td = tmp_path / "testdir" - self.assertFalse(td.exists()) + for d in td, td / "q", td / "s": + d.mkdir() + for f in ( + td / "p", + td / "q" / "w", + td / "q" / "x", + td / "r", + td / "s" / "y", + td / "s" / "z", + ): + f.write_bytes(b"") - @skipIf( + try: + rmtree(td) + except SkipTest as ex: + pytest.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + + assert not td.exists() + + @pytest.mark.skipif( sys.platform == "cygwin", - "Cygwin can't set the permissions that make the test meaningful.", + reason="Cygwin can't set the permissions that make the test meaningful.", ) - def test_rmtree_deletes_dir_with_readonly_files(self): + def test_deletes_dir_with_readonly_files(self, tmp_path): # Automatically works on Unix, but requires special handling on Windows. - # Not to be confused with what _tmpdir_to_force_permission_error sets up (see below). - with tempfile.TemporaryDirectory() as parent: - td = pathlib.Path(parent, "testdir") - for d in td, td / "sub": - d.mkdir() - for f in td / "x", td / "sub" / "y": - f.write_bytes(b"") - f.chmod(0) + # Not to be confused with what permission_error_tmpdir sets up (see below). + + td = tmp_path / "testdir" + + for d in td, td / "sub": + d.mkdir() + for f in td / "x", td / "sub" / "y": + f.write_bytes(b"") + f.chmod(0) + + try: + rmtree(td) + except SkipTest as ex: + self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + assert not td.exists() + + def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): + """rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + # Access the module through sys.modules so it is unambiguous which module's + # attribute we patch: the original git.util, not git.index.util even though + # git.index.util "replaces" git.util and is what "import git.util" gives us. + mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", True) + + # Disable common chmod functions so the callback can't fix the problem. + mocker.patch.object(os, "chmod") + mocker.patch.object(pathlib.Path, "chmod") + + # Now we can see how an intractable PermissionError is treated. + with pytest.raises(SkipTest): + rmtree(permission_error_tmpdir) + + def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir): + """rmtree does not wrap PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is false.""" + # See comments in test_wraps_perm_error_if_enabled for details about patching. + mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", False) + mocker.patch.object(os, "chmod") + mocker.patch.object(pathlib.Path, "chmod") + + with pytest.raises(PermissionError): + try: + rmtree(permission_error_tmpdir) + except SkipTest as ex: + pytest.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + + @pytest.mark.parametrize("hide_windows_known_errors", [False, True]) + def test_does_not_wrap_other_errors(self, mocker, file_not_found_tmpdir, hide_windows_known_errors): + # See comments in test_wraps_perm_error_if_enabled for details about patching. + mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) + mocker.patch.object(os, "chmod") + mocker.patch.object(pathlib.Path, "chmod") + + with pytest.raises(FileNotFoundError): try: - rmtree(td) + rmtree(file_not_found_tmpdir) except SkipTest as ex: self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") - self.assertFalse(td.exists()) - def test_rmtree_can_wrap_exceptions(self): - """rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" - with _tmpdir_to_force_permission_error() as td: - # Access the module through sys.modules so it is unambiguous which module's - # attribute we patch: the original git.util, not git.index.util even though - # git.index.util "replaces" git.util and is what "import git.util" gives us. - with mock.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", True): - # Disable common chmod functions so the callback can't fix the problem. - with mock.patch.object(os, "chmod"), mock.patch.object(pathlib.Path, "chmod"): - # Now we can see how an intractable PermissionError is treated. - with self.assertRaises(SkipTest): - rmtree(td) +class _Member: + """A member of an IterableList.""" - @ddt.data( - (False, PermissionError, _tmpdir_to_force_permission_error), - (False, FileNotFoundError, _tmpdir_for_file_not_found), - (True, FileNotFoundError, _tmpdir_for_file_not_found), - ) - def test_rmtree_does_not_wrap_unless_called_for(self, case): - """rmtree doesn't wrap non-PermissionError, nor if HIDE_WINDOWS_KNOWN_ERRORS is false.""" - hide_windows_known_errors, exception_type, tmpdir_context_factory = case - - with tmpdir_context_factory() as td: - # See comments in test_rmtree_can_wrap_exceptions regarding the patching done here. - with mock.patch.object( - sys.modules["git.util"], - "HIDE_WINDOWS_KNOWN_ERRORS", - hide_windows_known_errors, - ): - with mock.patch.object(os, "chmod"), mock.patch.object(pathlib.Path, "chmod"): - with self.assertRaises(exception_type): - try: - rmtree(td) - except SkipTest as ex: - self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + __slots__ = ("name",) + + def __init__(self, name): + self.name = name + + def __repr__(self): + return f"{type(self).__name__}({self.name!r})" + + +@ddt.ddt +class TestUtils(TestBase): + """Tests for utilities in :mod:`git.util` other than :func:`git.util.rmtree`.""" @ddt.data("HIDE_WINDOWS_KNOWN_ERRORS", "HIDE_WINDOWS_FREEZE_ERRORS") def test_env_vars_for_windows_tests(self, name): From c4da058ba99aafd16a01b0692b13bafb833969ac Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 09:43:41 -0400 Subject: [PATCH 0837/1790] Pull HIDE_WINDOWS_*_ERRORS tests out of TestUtils And make them pure pytest tests. This creates the TestEnvParsing class, which does not inherit from TestCase. Because unlike @ddt.data (and @ddt.idata) @pytest.mark.parametrize can be nested (applied multiple times, creating tests for all combinations), that is used for the HIDE_WINDOWS_*_ERRORS tests instead of subtests. Because there were not any other uses of subtests in the test suite, the pytest-subtest plugin is accordingly removed from test-requirements. It may be put back anytime in the future. --- test-requirements.txt | 1 - test/test_util.py | 82 +++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8cadfb836..7cfb977a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,5 +8,4 @@ pytest >= 7.3.1 pytest-cov pytest-instafail pytest-mock -pytest-subtests pytest-sugar diff --git a/test/test_util.py b/test/test_util.py index 784dc9bfc..c04f390d1 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -155,38 +155,26 @@ def test_does_not_wrap_other_errors(self, mocker, file_not_found_tmpdir, hide_wi self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") -class _Member: - """A member of an IterableList.""" - - __slots__ = ("name",) - - def __init__(self, name): - self.name = name - - def __repr__(self): - return f"{type(self).__name__}({self.name!r})" - +class TestEnvParsing: + """Tests for environment variable parsing logic in :mod:`git.util`.""" + + @staticmethod + def _run_parse(name, value): + command = [ + sys.executable, + "-c", + f"from git.util import {name}; print(repr({name}))", + ] + output = subprocess.check_output( + command, + env=None if value is None else dict(os.environ, **{name: value}), + text=True, + ) + return ast.literal_eval(output) -@ddt.ddt -class TestUtils(TestBase): - """Tests for utilities in :mod:`git.util` other than :func:`git.util.rmtree`.""" - - @ddt.data("HIDE_WINDOWS_KNOWN_ERRORS", "HIDE_WINDOWS_FREEZE_ERRORS") - def test_env_vars_for_windows_tests(self, name): - def run_parse(value): - command = [ - sys.executable, - "-c", - f"from git.util import {name}; print(repr({name}))", - ] - output = subprocess.check_output( - command, - env=None if value is None else dict(os.environ, **{name: value}), - text=True, - ) - return ast.literal_eval(output) - - for env_var_value, expected_truth_value in ( + @pytest.mark.parametrize( + "env_var_value, expected_truth_value", + [ (None, os.name == "nt"), # True on Windows when the environment variable is unset. ("", False), (" ", False), @@ -202,9 +190,35 @@ def run_parse(value): ("YES", os.name == "nt"), (" no ", False), (" yes ", os.name == "nt"), - ): - with self.subTest(env_var_value=env_var_value): - self.assertIs(run_parse(env_var_value), expected_truth_value) + ], + ) + @pytest.mark.parametrize( + "name", + [ + "HIDE_WINDOWS_KNOWN_ERRORS", + "HIDE_WINDOWS_FREEZE_ERRORS", + ], + ) + def test_env_vars_for_windows_tests(self, name, env_var_value, expected_truth_value): + actual_parsed_value = self._run_parse(name, env_var_value) + assert actual_parsed_value is expected_truth_value + + +class _Member: + """A member of an IterableList.""" + + __slots__ = ("name",) + + def __init__(self, name): + self.name = name + + def __repr__(self): + return f"{type(self).__name__}({self.name!r})" + + +@ddt.ddt +class TestUtils(TestBase): + """Tests for most utilities in :mod:`git.util`.""" _norm_cygpath_pairs = ( (R"foo\bar", "foo/bar"), From aa1799f0de449c50fb5e35ba0befb4ff2fc6f98c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 09:51:40 -0400 Subject: [PATCH 0838/1790] Remove file_not_found_tmpdir fixture for TestRmtree Since its presence doesn't make things any simpler or more elegant. (It was left over from a previous approach where it was used in some @ddt parameters.) --- test/test_util.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index c04f390d1..a02ca0cce 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -59,12 +59,6 @@ def permission_error_tmpdir(tmp_path): yield td -@pytest.fixture -def file_not_found_tmpdir(tmp_path): - """Fixture to test errors deleting a directory that are not due to permissions.""" - yield tmp_path / "testdir" # It is deliberately never created. - - class TestRmtree: """Tests for :func:`git.util.rmtree`.""" @@ -142,7 +136,9 @@ def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_ pytest.fail(f"rmtree unexpectedly attempts skip: {ex!r}") @pytest.mark.parametrize("hide_windows_known_errors", [False, True]) - def test_does_not_wrap_other_errors(self, mocker, file_not_found_tmpdir, hide_windows_known_errors): + def test_does_not_wrap_other_errors(self, tmp_path, mocker, hide_windows_known_errors): + file_not_found_tmpdir = tmp_path / "testdir" # It is deliberately never created. + # See comments in test_wraps_perm_error_if_enabled for details about patching. mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) mocker.patch.object(os, "chmod") From 2fb8c64a898e4c4bf93febadffa216c8a6ff2411 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 09:57:05 -0400 Subject: [PATCH 0839/1790] Move permission_error_tmpdir skips to test cases This moves test skipping logic from the permission_error_tmpdir fixture to the two TestRmtree test case methods that use it. This produces some duplication, which is undesirable, but it makes it so that which tests are skipped under what conditions is immediately clear, easy to identify as skipping logic (which is usually achieved by decoration), and clear in its relationship to the skipping logic associated with other TestRmtree test cases. --- test/test_util.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index a02ca0cce..1147676bc 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -46,11 +46,6 @@ @pytest.fixture def permission_error_tmpdir(tmp_path): """Fixture to test permissions errors situations where they are not overcome.""" - if sys.platform == "cygwin": - raise SkipTest("Cygwin can't set the permissions that make the test meaningful.") - if sys.version_info < (3, 8): - raise SkipTest("In 3.7, TemporaryDirectory doesn't clean up after weird permissions.") - td = tmp_path / "testdir" td.mkdir() (td / "x").write_bytes(b"") @@ -107,6 +102,14 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): assert not td.exists() + @pytest.mark.skipif( + sys.platform == "cygwin", + reason="Cygwin can't set the permissions that make the test meaningful.", + ) + @pytest.mark.skipif( + sys.version_info < (3, 8), + reason="In 3.7, TemporaryDirectory doesn't clean up after weird permissions.", + ) def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): """rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" # Access the module through sys.modules so it is unambiguous which module's @@ -122,6 +125,14 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): with pytest.raises(SkipTest): rmtree(permission_error_tmpdir) + @pytest.mark.skipif( + sys.platform == "cygwin", + reason="Cygwin can't set the permissions that make the test meaningful.", + ) + @pytest.mark.skipif( + sys.version_info < (3, 8), + reason="In 3.7, TemporaryDirectory doesn't clean up after weird permissions.", + ) def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir): """rmtree does not wrap PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is false.""" # See comments in test_wraps_perm_error_if_enabled for details about patching. From 1dccb8e3a90ad2227d0689171526212cd5740451 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 24 Oct 2023 11:16:31 -0400 Subject: [PATCH 0840/1790] Pull cygpath and decygpath tests out of TestUtils This turns them into pure pytest tests, in a new class TestCygpath. --- test/test_util.py | 98 ++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 1147676bc..f5d716d4c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -14,7 +14,7 @@ import sys import tempfile import time -from unittest import SkipTest, mock, skipUnless +from unittest import SkipTest, mock import ddt import pytest @@ -45,7 +45,7 @@ @pytest.fixture def permission_error_tmpdir(tmp_path): - """Fixture to test permissions errors situations where they are not overcome.""" + """Fixture to test permissions errors in situations where they are not overcome.""" td = tmp_path / "testdir" td.mkdir() (td / "x").write_bytes(b"") @@ -211,21 +211,9 @@ def test_env_vars_for_windows_tests(self, name, env_var_value, expected_truth_va assert actual_parsed_value is expected_truth_value -class _Member: - """A member of an IterableList.""" - - __slots__ = ("name",) - - def __init__(self, name): - self.name = name - - def __repr__(self): - return f"{type(self).__name__}({self.name!r})" - - -@ddt.ddt -class TestUtils(TestBase): - """Tests for most utilities in :mod:`git.util`.""" +@pytest.mark.skipif(sys.platform != "cygwin", reason="Paths specifically for Cygwin.") +class TestCygpath: + """Tests for :func:`git.util.cygpath` and :func:`git.util.decygpath`.""" _norm_cygpath_pairs = ( (R"foo\bar", "foo/bar"), @@ -248,54 +236,70 @@ class TestUtils(TestBase): (R"\\?\UNC\server\D$\Apps", "//server/D$/Apps"), ) - # FIXME: Mark only the /proc-prefixing cases xfail, somehow (or fix them). + # FIXME: Mark only the /proc-prefixing cases xfail (or fix them). @pytest.mark.xfail( reason="Many return paths prefixed /proc/cygdrive instead.", raises=AssertionError, ) - @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") - @ddt.idata(_norm_cygpath_pairs + _unc_cygpath_pairs) - def test_cygpath_ok(self, case): - wpath, cpath = case + @pytest.mark.parametrize("wpath, cpath", _norm_cygpath_pairs + _unc_cygpath_pairs) + def test_cygpath_ok(self, wpath, cpath): cwpath = cygpath(wpath) - self.assertEqual(cwpath, cpath, wpath) + assert cwpath == cpath, wpath @pytest.mark.xfail( reason=R'2nd example r".\bar" -> "bar" fails, returns "./bar"', raises=AssertionError, ) - @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") - @ddt.data( - (R"./bar", "bar"), - (R".\bar", "bar"), # FIXME: Mark only this one xfail, somehow (or fix it). - (R"../bar", "../bar"), - (R"..\bar", "../bar"), - (R"../bar/.\foo/../chu", "../bar/chu"), + @pytest.mark.parametrize( + "wpath, cpath", + [ + (R"./bar", "bar"), + (R".\bar", "bar"), # FIXME: Mark only this one xfail (or fix it). + (R"../bar", "../bar"), + (R"..\bar", "../bar"), + (R"../bar/.\foo/../chu", "../bar/chu"), + ], ) - def test_cygpath_norm_ok(self, case): - wpath, cpath = case + def test_cygpath_norm_ok(self, wpath, cpath): cwpath = cygpath(wpath) - self.assertEqual(cwpath, cpath or wpath, wpath) + assert cwpath == (cpath or wpath), wpath - @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") - @ddt.data( - R"C:", - R"C:Relative", - R"D:Apps\123", - R"D:Apps/123", - R"\\?\a:rel", - R"\\share\a:rel", + @pytest.mark.parametrize( + "wpath", + [ + R"C:", + R"C:Relative", + R"D:Apps\123", + R"D:Apps/123", + R"\\?\a:rel", + R"\\share\a:rel", + ], ) def test_cygpath_invalids(self, wpath): cwpath = cygpath(wpath) - self.assertEqual(cwpath, wpath.replace("\\", "/"), wpath) + assert cwpath == wpath.replace("\\", "/"), wpath - @skipUnless(sys.platform == "cygwin", "Paths specifically for Cygwin.") - @ddt.idata(_norm_cygpath_pairs) - def test_decygpath(self, case): - wpath, cpath = case + @pytest.mark.parametrize("wpath, cpath", _norm_cygpath_pairs) + def test_decygpath(self, wpath, cpath): wcpath = decygpath(cpath) - self.assertEqual(wcpath, wpath.replace("/", "\\"), cpath) + assert wcpath == wpath.replace("/", "\\"), cpath + + +class _Member: + """A member of an IterableList.""" + + __slots__ = ("name",) + + def __init__(self, name): + self.name = name + + def __repr__(self): + return f"{type(self).__name__}({self.name!r})" + + +@ddt.ddt +class TestUtils(TestBase): + """Tests for most utilities in :mod:`git.util`.""" def test_it_should_dashify(self): self.assertEqual("this-is-my-argument", dashify("this_is_my_argument")) From 487bc8aa71ebfdfa455c4037d0bbcfd51eb4997c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 25 Oct 2023 06:27:35 -0400 Subject: [PATCH 0841/1790] Start making TestCygpath xfail markings granular This marks only the one really expected failure case in in test_cygpath_norm_ok as xfail, preventing the others from giving "XPASS" results each time the tests are run. --- test/test_util.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index f5d716d4c..39dbe5e56 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -211,6 +211,11 @@ def test_env_vars_for_windows_tests(self, name, env_var_value, expected_truth_va assert actual_parsed_value is expected_truth_value +def _xfail_param(*values, **xfail_kwargs): + """Build a pytest.mark.parametrize parameter that carries an xfail mark.""" + return pytest.param(*values, marks=pytest.mark.xfail(**xfail_kwargs)) + + @pytest.mark.skipif(sys.platform != "cygwin", reason="Paths specifically for Cygwin.") class TestCygpath: """Tests for :func:`git.util.cygpath` and :func:`git.util.decygpath`.""" @@ -246,15 +251,11 @@ def test_cygpath_ok(self, wpath, cpath): cwpath = cygpath(wpath) assert cwpath == cpath, wpath - @pytest.mark.xfail( - reason=R'2nd example r".\bar" -> "bar" fails, returns "./bar"', - raises=AssertionError, - ) @pytest.mark.parametrize( "wpath, cpath", [ (R"./bar", "bar"), - (R".\bar", "bar"), # FIXME: Mark only this one xfail (or fix it). + _xfail_param(R".\bar", "bar", reason=R'Returns: "./bar"', raises=AssertionError), (R"../bar", "../bar"), (R"..\bar", "../bar"), (R"../bar/.\foo/../chu", "../bar/chu"), From c3d3d1ecd3ce8a2ab547a7f1d94590a676f36d18 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 25 Oct 2023 10:24:44 -0400 Subject: [PATCH 0842/1790] Finish making TestCygpath xfail markings granular This marks only the (many, but not all) really expected failure cases in test_cygpath_ok as xfail, preventing the others from giving "XPASS" results each time the tests are run. This finishes making the xfail markings in TestCygpath granular, in the sense that now there should be no "XPASS" results. However, the approach taken here might still benefit from future reorganization or refactoring. --- test/test_util.py | 78 ++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 39dbe5e56..a12d17db3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -216,37 +216,59 @@ def _xfail_param(*values, **xfail_kwargs): return pytest.param(*values, marks=pytest.mark.xfail(**xfail_kwargs)) -@pytest.mark.skipif(sys.platform != "cygwin", reason="Paths specifically for Cygwin.") -class TestCygpath: - """Tests for :func:`git.util.cygpath` and :func:`git.util.decygpath`.""" +_norm_cygpath_pairs = ( + (R"foo\bar", "foo/bar"), + (R"foo/bar", "foo/bar"), + (R"C:\Users", "/cygdrive/c/Users"), + (R"C:\d/e", "/cygdrive/c/d/e"), + ("C:\\", "/cygdrive/c/"), + (R"\\server\C$\Users", "//server/C$/Users"), + (R"\\server\C$", "//server/C$"), + ("\\\\server\\c$\\", "//server/c$/"), + (R"\\server\BAR/", "//server/BAR/"), + (R"D:/Apps", "/cygdrive/d/Apps"), + (R"D:/Apps\fOO", "/cygdrive/d/Apps/fOO"), + (R"D:\Apps/123", "/cygdrive/d/Apps/123"), +) - _norm_cygpath_pairs = ( - (R"foo\bar", "foo/bar"), - (R"foo/bar", "foo/bar"), - (R"C:\Users", "/cygdrive/c/Users"), - (R"C:\d/e", "/cygdrive/c/d/e"), - ("C:\\", "/cygdrive/c/"), - (R"\\server\C$\Users", "//server/C$/Users"), - (R"\\server\C$", "//server/C$"), - ("\\\\server\\c$\\", "//server/c$/"), - (R"\\server\BAR/", "//server/BAR/"), - (R"D:/Apps", "/cygdrive/d/Apps"), - (R"D:/Apps\fOO", "/cygdrive/d/Apps/fOO"), - (R"D:\Apps/123", "/cygdrive/d/Apps/123"), - ) +_unc_cygpath_pairs = ( + (R"\\?\a:\com", "/cygdrive/a/com"), + (R"\\?\a:/com", "/cygdrive/a/com"), + (R"\\?\UNC\server\D$\Apps", "//server/D$/Apps"), +) - _unc_cygpath_pairs = ( - (R"\\?\a:\com", "/cygdrive/a/com"), - (R"\\?\a:/com", "/cygdrive/a/com"), - (R"\\?\UNC\server\D$\Apps", "//server/D$/Apps"), +# Mapping of expected failures for the test_cygpath_ok test. +_cygpath_ok_xfails = { + # From _norm_cygpath_pairs: + (R"C:\Users", "/cygdrive/c/Users"): "/proc/cygdrive/c/Users", + (R"C:\d/e", "/cygdrive/c/d/e"): "/proc/cygdrive/c/d/e", + ("C:\\", "/cygdrive/c/"): "/proc/cygdrive/c/", + (R"\\server\BAR/", "//server/BAR/"): "//server/BAR", + (R"D:/Apps", "/cygdrive/d/Apps"): "/proc/cygdrive/d/Apps", + (R"D:/Apps\fOO", "/cygdrive/d/Apps/fOO"): "/proc/cygdrive/d/Apps/fOO", + (R"D:\Apps/123", "/cygdrive/d/Apps/123"): "/proc/cygdrive/d/Apps/123", + # From _unc_cygpath_pairs: + (R"\\?\a:\com", "/cygdrive/a/com"): "/proc/cygdrive/a/com", + (R"\\?\a:/com", "/cygdrive/a/com"): "/proc/cygdrive/a/com", +} + + +# Parameter sets for the test_cygpath_ok test. +_cygpath_ok_params = [ + ( + _xfail_param(*case, reason=f"Returns: {_cygpath_ok_xfails[case]!r}", raises=AssertionError) + if case in _cygpath_ok_xfails + else case ) + for case in _norm_cygpath_pairs + _unc_cygpath_pairs +] - # FIXME: Mark only the /proc-prefixing cases xfail (or fix them). - @pytest.mark.xfail( - reason="Many return paths prefixed /proc/cygdrive instead.", - raises=AssertionError, - ) - @pytest.mark.parametrize("wpath, cpath", _norm_cygpath_pairs + _unc_cygpath_pairs) + +@pytest.mark.skipif(sys.platform != "cygwin", reason="Paths specifically for Cygwin.") +class TestCygpath: + """Tests for :func:`git.util.cygpath` and :func:`git.util.decygpath`.""" + + @pytest.mark.parametrize("wpath, cpath", _cygpath_ok_params) def test_cygpath_ok(self, wpath, cpath): cwpath = cygpath(wpath) assert cwpath == cpath, wpath @@ -255,7 +277,7 @@ def test_cygpath_ok(self, wpath, cpath): "wpath, cpath", [ (R"./bar", "bar"), - _xfail_param(R".\bar", "bar", reason=R'Returns: "./bar"', raises=AssertionError), + _xfail_param(R".\bar", "bar", reason="Returns: './bar'", raises=AssertionError), (R"../bar", "../bar"), (R"..\bar", "../bar"), (R"../bar/.\foo/../chu", "../bar/chu"), From 7d98f94a60b6108709fc1155cd23c3224880ec94 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 27 Oct 2023 01:10:53 -0400 Subject: [PATCH 0843/1790] Let all TestRmtree tests run on 3.7 They are no longer using TemporaryDirectory (because they use the pytest tmp_path fixture instead), so the limitations of TemporaryDirectory on Python 3.7 are no longer relevant. --- test/test_util.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index a12d17db3..55f87ed9b 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -106,10 +106,6 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", ) - @pytest.mark.skipif( - sys.version_info < (3, 8), - reason="In 3.7, TemporaryDirectory doesn't clean up after weird permissions.", - ) def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): """rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" # Access the module through sys.modules so it is unambiguous which module's @@ -129,10 +125,6 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", ) - @pytest.mark.skipif( - sys.version_info < (3, 8), - reason="In 3.7, TemporaryDirectory doesn't clean up after weird permissions.", - ) def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir): """rmtree does not wrap PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is false.""" # See comments in test_wraps_perm_error_if_enabled for details about patching. From ba5bc45b3a570ef628dba97128ce4bfe9856d736 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 29 Oct 2023 03:34:03 -0400 Subject: [PATCH 0844/1790] Better explain the Windows and Unix cases Of PermissionError, in the rmtree tests. --- test/test_util.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 55f87ed9b..d345247b1 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -49,8 +49,13 @@ def permission_error_tmpdir(tmp_path): td = tmp_path / "testdir" td.mkdir() (td / "x").write_bytes(b"") - (td / "x").chmod(stat.S_IRUSR) # Set up PermissionError on Windows. - td.chmod(stat.S_IRUSR | stat.S_IXUSR) # Set up PermissionError on Unix. + + # Set up PermissionError on Windows, where we can't delete read-only files. + (td / "x").chmod(stat.S_IRUSR) + + # Set up PermissionError on Unix, where we can't delete files in read-only directories. + td.chmod(stat.S_IRUSR | stat.S_IXUSR) + yield td @@ -113,7 +118,7 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): # git.index.util "replaces" git.util and is what "import git.util" gives us. mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", True) - # Disable common chmod functions so the callback can't fix the problem. + # Disable common chmod functions so the callback can never fix the problem. mocker.patch.object(os, "chmod") mocker.patch.object(pathlib.Path, "chmod") From 454032cf5b783bca7d1e8b572e8a8944a5ef2036 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 3 Nov 2023 12:51:29 -0400 Subject: [PATCH 0845/1790] Make comments more consistent and clarify license This improves the consistency of top-of-module comments as follows: - All names of the current file are removed. Some included these while others didn't. In general, this can be useful information, which can remind readers and developers of what the file is and may even reduce mistakes. However, in GitPython, many modules inside git/ have the same name as other modules in other subdirectories of git/. So the presence of filenames would often be the same for multiple files, a condition that would be intensified if consistency were achieved by adding them everywhere. This instead removes them, which should (albeit slightly) decrease the risk of confusing modules that have the same name as each other. - All modules (.py files) inside git/ and test/, except for .py files that are entirely empty (without even comments) or are inside test/fixtures/, now have comments indicating the license and linking to it on opensource.org. Previously, some modules had this, while others did not. The comment about the license is short, and does not contain an explicit copyright statement. No new explicit copyright statements are added, but some top-of-modules already contained them, and this does not remove (nor update or otherwise modify) them. Although explicit copyright statements are not touched, all the license comments are modified, including where they had previously appeared, to say "the 3-Clause BSD License" instead of "the BSD License", since there is no specific license known as the "BSD License" (and both the 2-clause and 3-clause BSD licenses are very popular). This change should not be confused with #1662, which fixed an originally correct hyperlink that had come to redirect to a page about a different license. The change here does not change the link again. It makes the commented wording more specific, so that it is clear, even without looking at the link, which BSD license is being referred to. --- git/__init__.py | 5 ++--- git/cmd.py | 5 ++--- git/compat.py | 5 ++--- git/config.py | 5 ++--- git/db.py | 3 +++ git/diff.py | 5 ++--- git/exc.py | 5 ++--- git/index/__init__.py | 3 +++ git/index/base.py | 5 ++--- git/index/fun.py | 3 +++ git/index/typ.py | 3 +++ git/index/util.py | 3 +++ git/objects/__init__.py | 3 +++ git/objects/base.py | 5 ++--- git/objects/blob.py | 5 ++--- git/objects/commit.py | 5 ++--- git/objects/fun.py | 3 +++ git/objects/submodule/__init__.py | 3 +++ git/objects/submodule/base.py | 3 +++ git/objects/submodule/root.py | 3 +++ git/objects/submodule/util.py | 3 +++ git/objects/tag.py | 5 ++--- git/objects/tree.py | 5 ++--- git/objects/util.py | 5 ++--- git/refs/__init__.py | 4 ++++ git/refs/head.py | 3 +++ git/refs/log.py | 3 +++ git/refs/reference.py | 3 +++ git/refs/remote.py | 3 +++ git/refs/symbolic.py | 3 +++ git/refs/tag.py | 3 +++ git/remote.py | 5 ++--- git/repo/__init__.py | 3 +++ git/repo/base.py | 5 ++--- git/repo/fun.py | 3 +++ git/types.py | 4 ++-- git/util.py | 5 ++--- test/__init__.py | 5 ++--- test/lib/__init__.py | 5 ++--- test/lib/helper.py | 5 ++--- test/performance/lib.py | 3 +++ test/performance/test_commit.py | 4 ++-- test/performance/test_odb.py | 3 +++ test/performance/test_streams.py | 3 +++ test/test_actor.py | 5 ++--- test/test_base.py | 5 ++--- test/test_blob.py | 5 ++--- test/test_blob_filter.py | 3 +++ test/test_clone.py | 4 ++-- test/test_commit.py | 5 ++--- test/test_config.py | 5 ++--- test/test_db.py | 5 ++--- test/test_diff.py | 5 ++--- test/test_docs.py | 5 ++--- test/test_exc.py | 5 ++--- test/test_fun.py | 3 +++ test/test_git.py | 5 ++--- test/test_index.py | 5 ++--- test/test_installation.py | 4 ++-- test/test_quick_doc.py | 3 +++ test/test_reflog.py | 3 +++ test/test_refs.py | 5 ++--- test/test_remote.py | 5 ++--- test/test_repo.py | 6 ++---- test/test_stats.py | 5 ++--- test/test_submodule.py | 4 ++-- test/test_tree.py | 5 ++--- test/test_util.py | 5 ++--- test/tstrunner.py | 3 +++ 69 files changed, 167 insertions(+), 119 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 46d54a960..defc679cb 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -1,8 +1,7 @@ -# __init__.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ # flake8: noqa # @PydevCodeAnalysisIgnore diff --git a/git/cmd.py b/git/cmd.py index fd39a8eeb..c7c84c360 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1,8 +1,7 @@ -# cmd.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from __future__ import annotations diff --git a/git/compat.py b/git/compat.py index f17e52f7b..9a2116d44 100644 --- a/git/compat.py +++ b/git/compat.py @@ -1,8 +1,7 @@ -# compat.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Utilities to help provide compatibility with Python 3.""" diff --git a/git/config.py b/git/config.py index 2cb057021..2d207363f 100644 --- a/git/config.py +++ b/git/config.py @@ -1,8 +1,7 @@ -# config.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing module parser implementation able to properly read and write configuration files.""" diff --git a/git/db.py b/git/db.py index 9e278ea75..03b631084 100644 --- a/git/db.py +++ b/git/db.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Module with our own gitdb implementation - it uses the git command.""" from git.util import bin_to_hex, hex_to_bin diff --git a/git/diff.py b/git/diff.py index 275534bbf..25334512a 100644 --- a/git/diff.py +++ b/git/diff.py @@ -1,8 +1,7 @@ -# diff.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import re from git.cmd import handle_process_output diff --git a/git/exc.py b/git/exc.py index 124c5eeea..35008c29a 100644 --- a/git/exc.py +++ b/git/exc.py @@ -1,8 +1,7 @@ -# exc.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing all exceptions thrown throughout the git package.""" diff --git a/git/index/__init__.py b/git/index/__init__.py index f9a534ee7..9954b9e88 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Initialize the index package.""" # flake8: noqa diff --git a/git/index/base.py b/git/index/base.py index c2333a2c2..dbfa2c9bf 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1,8 +1,7 @@ -# base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from contextlib import ExitStack import datetime diff --git a/git/index/fun.py b/git/index/fun.py index a35990d6d..5dac62a63 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + # Standalone functions to accompany the index implementation and make it more versatile. # NOTE: Autodoc hates it if this is a docstring. diff --git a/git/index/typ.py b/git/index/typ.py index 9f57f067b..7011fd03d 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Module with additional types used by the index.""" from binascii import b2a_hex diff --git a/git/index/util.py b/git/index/util.py index 08e49d860..2c558e9d9 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Module containing index utilities.""" from functools import wraps diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 2a4a114c7..9ca430285 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Import all submodules' main classes into the package space.""" # flake8: noqa diff --git a/git/objects/base.py b/git/objects/base.py index 9f188a955..934fb40bc 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -1,8 +1,7 @@ -# base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from git.exc import WorkTreeRepositoryUnsupported from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex diff --git a/git/objects/blob.py b/git/objects/blob.py index f0d3181c2..6d7e859af 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -1,8 +1,7 @@ -# blob.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from mimetypes import guess_type from . import base diff --git a/git/objects/commit.py b/git/objects/commit.py index 04acb668b..7310d66b0 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -1,8 +1,7 @@ -# commit.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import datetime import re diff --git a/git/objects/fun.py b/git/objects/fun.py index 7756154be..6d8a23d35 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Module with functions which are supposed to be as fast as possible.""" from stat import S_ISDIR diff --git a/git/objects/submodule/__init__.py b/git/objects/submodule/__init__.py index 8edc13be4..b11b568f2 100644 --- a/git/objects/submodule/__init__.py +++ b/git/objects/submodule/__init__.py @@ -1,2 +1,5 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + # NOTE: Cannot import anything here as the top-level __init__ has to handle # our dependencies. diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 1516306ec..25c2d0fef 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from io import BytesIO import logging import os diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index cfcbb4cb7..d9d9f6d24 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from .base import Submodule, UpdateProgress from .util import find_first_remote_branch from git.exc import InvalidGitRepositoryError diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 3fc0b0b56..f8265798d 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + import git from git.exc import InvalidGitRepositoryError from git.config import GitConfigParser diff --git a/git/objects/tag.py b/git/objects/tag.py index 6eb1c8d90..500879d54 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -1,8 +1,7 @@ -# tag.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing all Object-based types.""" diff --git a/git/objects/tree.py b/git/objects/tree.py index 1be6f193e..4d94a5d24 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -1,8 +1,7 @@ -# tree.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from git.util import IterableList, join_path import git.diff as git_diff diff --git a/git/objects/util.py b/git/objects/util.py index 7af7fa0e5..9f42227d0 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -1,8 +1,7 @@ -# util.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Module for general utility functions.""" diff --git a/git/refs/__init__.py b/git/refs/__init__.py index 18ea2013c..3a82b9796 100644 --- a/git/refs/__init__.py +++ b/git/refs/__init__.py @@ -1,5 +1,9 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + # flake8: noqa # Import all modules in order, fix the names they require. + from .symbolic import * from .reference import * from .head import * diff --git a/git/refs/head.py b/git/refs/head.py index fa40943c6..fba195aaa 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from git.config import GitConfigParser, SectionConstraint from git.util import join_path from git.exc import GitCommandError diff --git a/git/refs/log.py b/git/refs/log.py index ebdaf04d1..aeebac48c 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from mmap import mmap import re import time as _time diff --git a/git/refs/reference.py b/git/refs/reference.py index f0eb6bfaa..c2ad13bd6 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from git.util import ( LazyMixin, IterableObj, diff --git a/git/refs/remote.py b/git/refs/remote.py index f26ee08fc..dd4117fa7 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + import os from git.util import join_path diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 99a60201f..84c2057e1 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from git.types import PathLike import os diff --git a/git/refs/tag.py b/git/refs/tag.py index d00adc121..a59a51337 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from .reference import Reference __all__ = ["TagReference", "Tag"] diff --git a/git/remote.py b/git/remote.py index ccf70a25c..4055dba2e 100644 --- a/git/remote.py +++ b/git/remote.py @@ -1,8 +1,7 @@ -# remote.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Module implementing a remote object allowing easy access to git remotes.""" diff --git a/git/repo/__init__.py b/git/repo/__init__.py index f1eac3311..c01a1e034 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Initialize the Repo package.""" # flake8: noqa diff --git a/git/repo/base.py b/git/repo/base.py index 4790ea4e7..ebe72d1e7 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1,8 +1,7 @@ -# base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from __future__ import annotations diff --git a/git/repo/fun.py b/git/repo/fun.py index 29a899ea8..ee831332f 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Module with general repository-related functions.""" from __future__ import annotations diff --git a/git/types.py b/git/types.py index 2709bbf34..6f2b7c513 100644 --- a/git/types.py +++ b/git/types.py @@ -1,5 +1,5 @@ -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ # flake8: noqa diff --git a/git/util.py b/git/util.py index bd1fbe247..e5d03d157 100644 --- a/git/util.py +++ b/git/util.py @@ -1,8 +1,7 @@ -# util.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from abc import abstractmethod import contextlib diff --git a/test/__init__.py b/test/__init__.py index a3d514523..fbaebcd3b 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,4 @@ -# __init__.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/lib/__init__.py b/test/lib/__init__.py index 299317c0b..cc1e48483 100644 --- a/test/lib/__init__.py +++ b/test/lib/__init__.py @@ -1,8 +1,7 @@ -# __init__.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ # flake8: noqa import inspect diff --git a/test/lib/helper.py b/test/lib/helper.py index 8725cd13f..387686327 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -1,8 +1,7 @@ -# helper.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib from functools import wraps diff --git a/test/performance/lib.py b/test/performance/lib.py index 2b2a632d9..ceee6c2a1 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Support library for tests.""" import logging diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index fad0641be..9e136a6c1 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -1,7 +1,7 @@ # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ """Performance tests for commits (iteration, traversal, and serialization).""" diff --git a/test/performance/test_odb.py b/test/performance/test_odb.py index 70934ad6b..00e245fb7 100644 --- a/test/performance/test_odb.py +++ b/test/performance/test_odb.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Performance tests for object store.""" import sys diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index 619126921..9ee7cf5e2 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Performance tests for data streaming.""" import os diff --git a/test/test_actor.py b/test/test_actor.py index 80b93d7bc..caf095739 100644 --- a/test/test_actor.py +++ b/test/test_actor.py @@ -1,8 +1,7 @@ -# test_actor.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from test.lib import TestBase from git import Actor diff --git a/test/test_base.py b/test/test_base.py index e4704c7d8..725562c10 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -1,8 +1,7 @@ -# test_base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import os import sys diff --git a/test/test_blob.py b/test/test_blob.py index 692522b52..ff59c67ea 100644 --- a/test/test_blob.py +++ b/test/test_blob.py @@ -1,8 +1,7 @@ -# test_blob.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from test.lib import TestBase from git import Blob diff --git a/test/test_blob_filter.py b/test/test_blob_filter.py index ad4f0e7ff..5cc6b48c9 100644 --- a/test/test_blob_filter.py +++ b/test/test_blob_filter.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Test the blob filter.""" from pathlib import Path diff --git a/test/test_clone.py b/test/test_clone.py index 7624b317b..dcab7ad6f 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,5 +1,5 @@ -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from pathlib import Path import re diff --git a/test/test_commit.py b/test/test_commit.py index 1327616ed..b6fb09aef 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -1,8 +1,7 @@ -# test_commit.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import copy from datetime import datetime diff --git a/test/test_config.py b/test/test_config.py index 63fbc61e6..0e1bba08a 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,8 +1,7 @@ -# test_config.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import glob import io diff --git a/test/test_db.py b/test/test_db.py index d59aa6cc0..de093cbd8 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,8 +1,7 @@ -# test_db.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from git.db import GitCmdObjectDB from git.exc import BadObject diff --git a/test/test_diff.py b/test/test_diff.py index 50b96efff..1678e737d 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -1,8 +1,7 @@ -# test_diff.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ddt import shutil diff --git a/test/test_docs.py b/test/test_docs.py index 394b58b5f..2f4b2e8d8 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -1,8 +1,7 @@ -# test_docs.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import os import sys diff --git a/test/test_exc.py b/test/test_exc.py index ad43695b3..62bb4fb6e 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -1,8 +1,7 @@ -# test_exc.py # Copyright (C) 2008, 2009, 2016 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import re diff --git a/test/test_fun.py b/test/test_fun.py index 0015b30c6..566bc9aae 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from io import BytesIO from stat import S_IFDIR, S_IFREG, S_IFLNK, S_IXUSR from os import stat diff --git a/test/test_git.py b/test/test_git.py index 06f8f5c97..c2fdf8feb 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -1,8 +1,7 @@ -# test_git.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import inspect import logging diff --git a/test/test_index.py b/test/test_index.py index 3357dc880..3e9e6124d 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1,8 +1,7 @@ -# test_index.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO import os diff --git a/test/test_installation.py b/test/test_installation.py index e7774d29d..8f8c1adf2 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -1,5 +1,5 @@ -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ast import os diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 13b587bd5..504dca237 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from test.lib import TestBase from test.lib.helper import with_rw_directory diff --git a/test/test_reflog.py b/test/test_reflog.py index d5173d2f4..625466d40 100644 --- a/test/test_reflog.py +++ b/test/test_reflog.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + import os import tempfile diff --git a/test/test_refs.py b/test/test_refs.py index ae07ce421..6ee385007 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -1,8 +1,7 @@ -# test_refs.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from itertools import chain from pathlib import Path diff --git a/test/test_remote.py b/test/test_remote.py index 8205c0bcd..12de18476 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -1,8 +1,7 @@ -# test_remote.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import random import tempfile diff --git a/test/test_repo.py b/test/test_repo.py index 1ba85acf9..e77bf2503 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1,9 +1,7 @@ -# test_repo.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ - +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import glob import io from io import BytesIO diff --git a/test/test_stats.py b/test/test_stats.py index 335ce483b..4efb6f313 100644 --- a/test/test_stats.py +++ b/test/test_stats.py @@ -1,8 +1,7 @@ -# test_stats.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from test.lib import TestBase, fixture from git import Stats diff --git a/test/test_submodule.py b/test/test_submodule.py index f63db1495..f92c0e4a4 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1,5 +1,5 @@ -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib import os diff --git a/test/test_tree.py b/test/test_tree.py index 5fc98e40c..7713413a6 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -1,8 +1,7 @@ -# test_tree.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO diff --git a/test/test_util.py b/test/test_util.py index d345247b1..4fb30f77f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,8 +1,7 @@ -# test_util.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # -# This module is part of GitPython and is released under -# the BSD License: https://opensource.org/license/bsd-3-clause/ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ast from datetime import datetime diff --git a/test/tstrunner.py b/test/tstrunner.py index 8613538eb..fc9a59c8c 100644 --- a/test/tstrunner.py +++ b/test/tstrunner.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Hook for MonkeyType (see PR #1188).""" import unittest From 7387dab1e5a7e93187674c8ea9b126210fd2f535 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 3 Nov 2023 13:15:52 -0400 Subject: [PATCH 0846/1790] Update README to clarify license name The license in README.md was named as "New BSD License", which is in practice unambiguous, but no longer as readily recognized as is "3-Clause BSD License". This updates the wording to call it by the latter name, for that reason and for consistency with the previous commit's changes to top-of-module comments, while also noting parenthetically that it is also called the New BSD License. The main reason for retaining "New BSD License" parenthetically is to prevent anyone from being confused into thinking this change is in any way associated with a change to the license, or that any license change has occurred or is forthcoming. (This does *not* change how GitPython is actually licensed in any way.) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 889ea635f..0e020a5fe 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ gpg --edit-key 4C08421980C9 ### LICENSE -[New BSD License](https://opensource.org/license/bsd-3-clause/). See the [LICENSE file][license]. +[3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md [license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE From 209162a4c53afe65054dd7ecdd06774e6196861f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 3 Nov 2023 13:51:14 -0400 Subject: [PATCH 0847/1790] Pass more specific license argument in setup.py This uses "BSD-3-Clause" instead of "BSD" as the "license" metadata in setup.py. "BSD-3-Clause" is the SPDX license identifier for the BSD 3-Clause "New" or "Revised" License (the license GitPython uses): https://spdx.org/licenses/BSD-3-Clause.html There is no requirement to use an SPDX license identifier here, but it is one of the common approaches, and it has the advantage of making unambiguously clear, when a package is published on PyPI, exactly what license it uses. In contrast, the license-related *classifier* is unchanged, since no more specific classfifier than what is in use now is currently available. The combination should result in License: BSD License (BSD-3-Clause) being shown under "Meta" on PyPI, as of the next PyPI release. This can be seen in other projects that use this combination of license keyword argument and license-related classifier, such as: https://pypi.org/project/flask-restx/ --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90df8d7ea..f40f7280c 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def _stamp_version(filename: str) -> None: description="GitPython is a Python library used to interact with Git repositories", author="Sebastian Thiel, Michael Trier", author_email="byronimo@gmail.com, mtrier@gmail.com", - license="BSD", + license="BSD-3-Clause", url="https://github.com/gitpython-developers/GitPython", packages=find_packages(exclude=["test", "test.*"]), include_package_data=True, From c01fe76f2351d06671296bffcd98a3f50543d785 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 29 Oct 2023 16:34:23 -0400 Subject: [PATCH 0848/1790] Deprecate compat.is_, rewriting all uses Major changes: - Add a comment in the git.compat module above the definitions of is_win, is_posix, and is_darwin stating that they are deprecated and recommending that os.name (or, where applicable, sys.platform) be used directly instead for clarity (and sometimes accuracy). - Remove all uses of is_win and is_posix in git/ and test/, replacing them with `os.name == "nt"` and `os.name == "posix"`, respectively. There were no uses of is_darwin to be replaced. Although it had been used at one time, the last reference to it appears to have been removed in 4545762 (#1295). This doesn't emit a DeprecationWarning when those attributes are accessed in the git.compat module. (That might be valuable thing to do in the future, if the git.compat module is to remain non-deprecated overall.) Two related less consequential changes are also included: - Improve ordering and grouping of imports, in modules where they were already being changed as a result of no longer needing to import is_ (usually is_win) from git.compat. - Make minor revisions to a few comments, for readability. (This is in addition to somewhat more substantial revisions of comments where rewording was related to replacing uses of is_.) --- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- git/cmd.py | 64 ++++++++++++++--------------- git/compat.py | 11 +++++ git/config.py | 19 +++------ git/index/fun.py | 22 ++++------ git/index/util.py | 8 +--- git/objects/submodule/base.py | 7 ++-- git/repo/base.py | 22 ++++------ git/util.py | 36 ++++++++++------ test/lib/helper.py | 10 ++--- test/test_base.py | 3 +- test/test_git.py | 10 +++-- test/test_index.py | 9 ++-- test/test_installation.py | 3 +- test/test_submodule.py | 15 +++---- test/test_util.py | 3 +- 17 files changed, 118 insertions(+), 128 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 89c03a394..96728e51a 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -72,7 +72,7 @@ jobs: python --version python -c 'import sys; print(sys.platform)' python -c 'import os; print(os.name)' - python -c 'import git; print(git.compat.is_win)' + python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. - name: Test with pytest run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 2dd97183b..35fecf16c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -63,7 +63,7 @@ jobs: python --version python -c 'import sys; print(sys.platform)' python -c 'import os; print(os.name)' - python -c 'import git; print(git.compat.is_win)' + python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. - name: Check types with mypy run: | diff --git a/git/cmd.py b/git/cmd.py index fd39a8eeb..f052c72cf 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -17,19 +17,21 @@ import threading from textwrap import dedent -from git.compat import ( - defenc, - force_bytes, - safe_decode, - is_posix, - is_win, +from git.compat import defenc, force_bytes, safe_decode +from git.exc import ( + CommandError, + GitCommandError, + GitCommandNotFound, + UnsafeOptionError, + UnsafeProtocolError, ) -from git.exc import CommandError -from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present, patch_env - -from .exc import GitCommandError, GitCommandNotFound, UnsafeOptionError, UnsafeProtocolError -from .util import ( +from git.util import ( LazyMixin, + cygpath, + expand_path, + is_cygwin_git, + patch_env, + remove_password_if_present, stream_copy, ) @@ -180,14 +182,13 @@ def pump_stream( t.start() threads.append(t) - ## FIXME: Why Join?? Will block if `stdin` needs feeding... - # + # FIXME: Why join? Will block if stdin needs feeding... for t in threads: t.join(timeout=kill_after_timeout) if t.is_alive(): if isinstance(process, Git.AutoInterrupt): process._terminate() - else: # Don't want to deal with the other case + else: # Don't want to deal with the other case. raise RuntimeError( "Thread join() timed out in cmd.handle_process_output()." f" kill_after_timeout={kill_after_timeout} seconds" @@ -197,11 +198,11 @@ def pump_stream( "error: process killed because it timed out." f" kill_after_timeout={kill_after_timeout} seconds" ) if not decode_streams and isinstance(p_stderr, BinaryIO): - # Assume stderr_handler needs binary input + # Assume stderr_handler needs binary input. error_str = cast(str, error_str) error_str = error_str.encode() # We ignore typing on the next line because mypy does not like - # the way we inferred that stderr takes str or bytes + # the way we inferred that stderr takes str or bytes. stderr_handler(error_str) # type: ignore if finalizer: @@ -228,14 +229,12 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ## -- End Utilities -- @} -# value of Windows process creation flag taken from MSDN -CREATE_NO_WINDOW = 0x08000000 - -## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, -# see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal -PROC_CREATIONFLAGS = ( - CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP if is_win else 0 # type: ignore[attr-defined] -) # mypy error if not Windows. +if os.name == "nt": + # CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards. See: + # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal + PROC_CREATIONFLAGS = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP +else: + PROC_CREATIONFLAGS = 0 class Git(LazyMixin): @@ -551,7 +550,7 @@ def _terminate(self) -> None: # For some reason, providing None for stdout/stderr still prints something. This is why # we simply use the shell and redirect to nul. Slower than CreateProcess. The question # is whether we really want to see all these messages. It's annoying no matter what. - if is_win: + if os.name == "nt": call( ("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), shell=True, @@ -967,7 +966,7 @@ def execute( if inline_env is not None: env.update(inline_env) - if is_win: + if os.name == "nt": cmd_not_found_exception = OSError if kill_after_timeout is not None: raise GitCommandError( @@ -999,11 +998,11 @@ def execute( env=env, cwd=cwd, bufsize=-1, - stdin=istream or DEVNULL, + stdin=(istream or DEVNULL), stderr=PIPE, stdout=stdout_sink, shell=shell, - close_fds=is_posix, # Unsupported on Windows. + close_fds=(os.name == "posix"), # Unsupported on Windows. universal_newlines=universal_newlines, creationflags=PROC_CREATIONFLAGS, **subprocess_kwargs, @@ -1073,7 +1072,7 @@ def kill_process(pid: int) -> None: ) if not universal_newlines: stderr_value = stderr_value.encode(defenc) - # strip trailing "\n" + # Strip trailing "\n". if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore stdout_value = stdout_value[:-1] if stderr_value.endswith(newline): # type: ignore @@ -1147,11 +1146,11 @@ def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: """ old_env = {} for key, value in kwargs.items(): - # set value if it is None + # Set value if it is None. if value is not None: old_env[key] = self._environment.get(key) self._environment[key] = value - # remove key from environment if its value is None + # Remove key from environment if its value is None. elif key in self._environment: old_env[key] = self._environment[key] del self._environment[key] @@ -1330,7 +1329,8 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: :return: (hex_sha, type_string, size_as_int) :raise ValueError: If the header contains indication for an error due to - incorrect input sha""" + incorrect input sha + """ tokens = header_line.split() if len(tokens) != 3: if not tokens: diff --git a/git/compat.py b/git/compat.py index f17e52f7b..545bf65b4 100644 --- a/git/compat.py +++ b/git/compat.py @@ -34,9 +34,20 @@ # --------------------------------------------------------------------------- +# DEPRECATED attributes providing shortcuts to operating system checks based on os.name. +# +# - is_win and is_posix are deprecated because it is clearer, and helps avoid bugs, to +# write out the os.name checks explicitly. For example, is_win is False on Cygwin, but +# is often assumed to be True. +# +# - is_darwin is deprecated because it is always False on all systems, as os.name is +# never "darwin". For macOS, you can check for sys.platform == "darwin". (As on other +# Unix-like systems, os.name == "posix" on macOS. This is also the case on Cygwin.) +# is_win: bool = os.name == "nt" is_posix = os.name == "posix" is_darwin = os.name == "darwin" + defenc = sys.getfilesystemencoding() diff --git a/git/config.py b/git/config.py index 2cb057021..0846bbf94 100644 --- a/git/config.py +++ b/git/config.py @@ -7,28 +7,21 @@ """Module containing module parser implementation able to properly read and write configuration files.""" -import sys import abc +import configparser as cp +import fnmatch from functools import wraps import inspect from io import BufferedReader, IOBase import logging import os +import os.path as osp import re -import fnmatch - -from git.compat import ( - defenc, - force_text, - is_win, -) +import sys +from git.compat import defenc, force_text from git.util import LockFile -import os.path as osp - -import configparser as cp - # typing------------------------------------------------------- from typing import ( @@ -250,7 +243,7 @@ def items_all(self) -> List[Tuple[str, List[_T]]]: def get_config_path(config_level: Lit_config_levels) -> str: # We do not support an absolute path of the gitconfig on Windows. # Use the global config instead. - if is_win and config_level == "system": + if os.name == "nt" and config_level == "system": config_level = "global" if config_level == "system": diff --git a/git/index/fun.py b/git/index/fun.py index a35990d6d..fe8f373d8 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -2,8 +2,9 @@ # NOTE: Autodoc hates it if this is a docstring. from io import BytesIO -from pathlib import Path import os +import os.path as osp +from pathlib import Path from stat import ( S_IFDIR, S_IFLNK, @@ -16,26 +17,17 @@ import subprocess from git.cmd import PROC_CREATIONFLAGS, handle_process_output -from git.compat import ( - defenc, - force_text, - force_bytes, - is_posix, - is_win, - safe_decode, -) -from git.exc import UnmergedEntriesError, HookExecutionError +from git.compat import defenc, force_bytes, force_text, safe_decode +from git.exc import HookExecutionError, UnmergedEntriesError from git.objects.fun import ( - tree_to_stream, traverse_tree_recursive, traverse_trees_recursive, + tree_to_stream, ) from git.util import IndexFileSHA1Writer, finalize_process from gitdb.base import IStream from gitdb.typ import str_tree_type -import os.path as osp - from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT from .util import pack, unpack @@ -96,7 +88,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: env["GIT_EDITOR"] = ":" cmd = [hp] try: - if is_win and not _has_file_extension(hp): + if os.name == "nt" and not _has_file_extension(hp): # Windows only uses extensions to determine how to open files # (doesn't understand shebangs). Try using bash to run the hook. relative_hp = Path(hp).relative_to(index.repo.working_dir).as_posix() @@ -108,7 +100,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=index.repo.working_dir, - close_fds=is_posix, + close_fds=(os.name == "posix"), creationflags=PROC_CREATIONFLAGS, ) except Exception as ex: diff --git a/git/index/util.py b/git/index/util.py index 08e49d860..5c2270abd 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -2,15 +2,11 @@ from functools import wraps import os +import os.path as osp import struct import tempfile from types import TracebackType -from git.compat import is_win - -import os.path as osp - - # typing ---------------------------------------------------------------------- from typing import Any, Callable, TYPE_CHECKING, Optional, Type @@ -58,7 +54,7 @@ def __exit__( exc_tb: Optional[TracebackType], ) -> bool: if osp.isfile(self.tmp_file_path): - if is_win and osp.exists(self.file_path): + if os.name == "nt" and osp.exists(self.file_path): os.remove(self.file_path) os.rename(self.tmp_file_path, self.file_path) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 1516306ec..d88524ecb 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -7,7 +7,7 @@ import git from git.cmd import Git -from git.compat import defenc, is_win +from git.compat import defenc from git.config import GitConfigParser, SectionConstraint, cp from git.exc import ( BadName, @@ -353,9 +353,8 @@ def _write_git_file_and_module_config(cls, working_tree_dir: PathLike, module_ab """ git_file = osp.join(working_tree_dir, ".git") rela_path = osp.relpath(module_abspath, start=working_tree_dir) - if is_win: - if osp.isfile(git_file): - os.remove(git_file) + if os.name == "nt" and osp.isfile(git_file): + os.remove(git_file) with open(git_file, "wb") as fp: fp.write(("gitdir: %s" % rela_path).encode(defenc)) diff --git a/git/repo/base.py b/git/repo/base.py index 4790ea4e7..bbbf242db 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -6,24 +6,21 @@ from __future__ import annotations +import gc import logging import os +import os.path as osp +from pathlib import Path import re import shlex import warnings -from pathlib import Path - +import gitdb from gitdb.db.loose import LooseObjectDB - from gitdb.exc import BadObject from git.cmd import Git, handle_process_output -from git.compat import ( - defenc, - safe_decode, - is_win, -) +from git.compat import defenc, safe_decode from git.config import GitConfigParser from git.db import GitCmdObjectDB from git.exc import ( @@ -43,7 +40,6 @@ expand_path, remove_password_if_present, ) -import os.path as osp from .fun import ( rev_parse, @@ -52,8 +48,6 @@ touch, find_worktree_git_dir, ) -import gc -import gitdb # typing ------------------------------------------------------ @@ -322,10 +316,10 @@ def close(self) -> None: # they are collected by the garbage collector, thus preventing deletion. # TODO: Find these references and ensure they are closed and deleted # synchronously rather than forcing a gc collection. - if is_win: + if os.name == "nt": gc.collect() gitdb.util.mman.collect() - if is_win: + if os.name == "nt": gc.collect() def __eq__(self, rhs: object) -> bool: @@ -571,7 +565,7 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa git_dir = self.git_dir # We do not support an absolute path of the gitconfig on Windows. # Use the global config instead. - if is_win and config_level == "system": + if os.name == "nt" and config_level == "system": config_level = "global" if config_level == "system": diff --git a/git/util.py b/git/util.py index bd1fbe247..be267fc9f 100644 --- a/git/util.py +++ b/git/util.py @@ -22,8 +22,6 @@ from urllib.parse import urlsplit, urlunsplit import warnings -from .compat import is_win - # typing --------------------------------------------------------- from typing import ( @@ -110,7 +108,18 @@ log = logging.getLogger(__name__) -def _read_env_flag(name: str, default: bool) -> bool: +def _read_win_env_flag(name: str, default: bool) -> bool: + """Read a boolean flag from an environment variable on Windows. + + :return: + On Windows, the flag, or the ``default`` value if absent or ambiguous. + On all other operating systems, ``False``. + + :note: This only accesses the environment on Windows. + """ + if os.name != "nt": + return False + try: value = os.environ[name] except KeyError: @@ -134,8 +143,8 @@ def _read_env_flag(name: str, default: bool) -> bool: #: We need an easy way to see if Appveyor TCs start failing, #: so the errors marked with this var are considered "acknowledged" ones, awaiting remedy, #: till then, we wish to hide them. -HIDE_WINDOWS_KNOWN_ERRORS = is_win and _read_env_flag("HIDE_WINDOWS_KNOWN_ERRORS", True) -HIDE_WINDOWS_FREEZE_ERRORS = is_win and _read_env_flag("HIDE_WINDOWS_FREEZE_ERRORS", True) +HIDE_WINDOWS_KNOWN_ERRORS = _read_win_env_flag("HIDE_WINDOWS_KNOWN_ERRORS", True) +HIDE_WINDOWS_FREEZE_ERRORS = _read_win_env_flag("HIDE_WINDOWS_FREEZE_ERRORS", True) # { Utility Methods @@ -220,7 +229,7 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: def rmfile(path: PathLike) -> None: """Ensure file deleted also on *Windows* where read-only files need special treatment.""" if osp.isfile(path): - if is_win: + if os.name == "nt": os.chmod(path, 0o777) os.remove(path) @@ -260,7 +269,7 @@ def join_path(a: PathLike, *p: PathLike) -> PathLike: return path -if is_win: +if os.name == "nt": def to_native_path_windows(path: PathLike) -> PathLike: path = str(path) @@ -308,9 +317,12 @@ def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: def _get_exe_extensions() -> Sequence[str]: PATHEXT = os.environ.get("PATHEXT", None) - return ( - tuple(p.upper() for p in PATHEXT.split(os.pathsep)) if PATHEXT else (".BAT", "COM", ".EXE") if is_win else ("") - ) + if PATHEXT: + return tuple(p.upper() for p in PATHEXT.split(os.pathsep)) + elif os.name == "nt": + return (".BAT", "COM", ".EXE") + else: + return () def py_where(program: str, path: Optional[PathLike] = None) -> List[str]: @@ -418,8 +430,8 @@ def is_cygwin_git(git_executable: PathLike) -> bool: def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - if is_win: - # is_win is only True on native Windows systems. On Cygwin, os.name == "posix". + if os.name == "nt": + # This is Windows-native Python, since Cygwin has os.name == "posix". return False if git_executable is None: diff --git a/test/lib/helper.py b/test/lib/helper.py index 8725cd13f..950ca3c0b 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -10,17 +10,15 @@ import io import logging import os +import os.path as osp import tempfile import textwrap import time import unittest -from git.compat import is_win -from git.util import rmtree, cwd import gitdb -import os.path as osp - +from git.util import rmtree, cwd TestCase = unittest.TestCase SkipTest = unittest.SkipTest @@ -177,7 +175,7 @@ def git_daemon_launched(base_path, ip, port): gd = None try: - if is_win: + if os.name == "nt": # On MINGW-git, daemon exists in Git\mingw64\libexec\git-core\, # but if invoked as 'git daemon', it detaches from parent `git` cmd, # and then CANNOT DIE! @@ -201,7 +199,7 @@ def git_daemon_launched(base_path, ip, port): as_process=True, ) # Yes, I know... fortunately, this is always going to work if sleep time is just large enough. - time.sleep(0.5 * (1 + is_win)) + time.sleep(1.0 if os.name == "nt" else 0.5) except Exception as ex: msg = textwrap.dedent( """ diff --git a/test/test_base.py b/test/test_base.py index e4704c7d8..302d696fe 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -11,7 +11,6 @@ from git import Repo from git.objects import Blob, Tree, Commit, TagObject -from git.compat import is_win from git.objects.util import get_object_type_by_name from test.lib import TestBase as _TestBase, with_rw_repo, with_rw_and_rw_remote_repo from git.util import hex_to_bin, HIDE_WINDOWS_FREEZE_ERRORS @@ -131,7 +130,7 @@ def test_add_unicode(self, rw_repo): with open(file_path, "wb") as fp: fp.write(b"something") - if is_win: + if os.name == "nt": # On Windows, there is no way this works, see images on: # https://github.com/gitpython-developers/GitPython/issues/147#issuecomment-68881897 # Therefore, it must be added using the Python implementation. diff --git a/test/test_git.py b/test/test_git.py index 06f8f5c97..b85be6359 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -23,7 +23,6 @@ import ddt from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd -from git.compat import is_win from git.util import cwd, finalize_process from test.lib import TestBase, fixture_path, with_rw_directory @@ -139,7 +138,7 @@ def test_it_executes_git_and_returns_result(self): def test_it_executes_git_not_from_cwd(self): with TemporaryDirectory() as tmpdir: - if is_win: + if os.name == "nt": # Copy an actual binary executable that is not git. other_exe_path = os.path.join(os.getenv("WINDIR"), "system32", "hostname.exe") impostor_path = os.path.join(tmpdir, "git.exe") @@ -154,7 +153,10 @@ def test_it_executes_git_not_from_cwd(self): with cwd(tmpdir): self.assertRegex(self.git.execute(["git", "version"]), r"^git version\b") - @skipUnless(is_win, "The regression only affected Windows, and this test logic is OS-specific.") + @skipUnless( + os.name == "nt", + "The regression only affected Windows, and this test logic is OS-specific.", + ) def test_it_avoids_upcasing_unrelated_environment_variable_names(self): old_name = "28f425ca_d5d8_4257_b013_8d63166c8158" if old_name == old_name.upper(): @@ -268,7 +270,7 @@ def test_refresh(self): self.assertRaises(GitCommandNotFound, refresh, "yada") # Test a good path refresh. - which_cmd = "where" if is_win else "command -v" + which_cmd = "where" if os.name == "nt" else "command -v" path = os.popen("{0} git".format(which_cmd)).read().strip().split("\n")[0] refresh(path) diff --git a/test/test_index.py b/test/test_index.py index 3357dc880..b9d1e78ab 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -25,7 +25,6 @@ GitCommandError, CheckoutError, ) -from git.compat import is_win from git.exc import HookExecutionError, InvalidGitRepositoryError from git.index.fun import hook_path from git.index.typ import BaseIndexEntry, IndexEntry @@ -43,9 +42,9 @@ def _found_in(cmd, directory): return path and Path(path).parent == Path(directory) -is_win_without_bash = is_win and not shutil.which("bash.exe") +is_win_without_bash = os.name == "nt" and not shutil.which("bash.exe") -is_win_with_wsl_bash = is_win and _found_in( +is_win_with_wsl_bash = os.name == "nt" and _found_in( cmd="bash.exe", directory=Path(os.getenv("WINDIR")) / "System32", ) @@ -614,7 +613,7 @@ def mixed_iterator(): self.assertNotEqual(entries[0].hexsha, null_hex_sha) # Add symlink. - if not is_win: + if os.name != "nt": for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): basename = "my_real_symlink" @@ -672,7 +671,7 @@ def mixed_iterator(): index.checkout(fake_symlink_path) # On Windows, we will never get symlinks. - if is_win: + if os.name == "nt": # Symlinks should contain the link as text (which is what a # symlink actually is). with open(fake_symlink_path, "rt") as fd: diff --git a/test/test_installation.py b/test/test_installation.py index e7774d29d..8e5f55d28 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -6,7 +6,6 @@ import subprocess import sys -from git.compat import is_win from test.lib import TestBase from test.lib.helper import with_rw_directory @@ -15,7 +14,7 @@ class TestInstallation(TestBase): def setUp_venv(self, rw_dir): self.venv = rw_dir subprocess.run([sys.executable, "-m", "venv", self.venv], stdout=subprocess.PIPE) - bin_name = "Scripts" if is_win else "bin" + bin_name = "Scripts" if os.name == "nt" else "bin" self.python = os.path.join(self.venv, bin_name, "python") self.pip = os.path.join(self.venv, bin_name, "pip") self.sources = os.path.join(self.venv, "src") diff --git a/test/test_submodule.py b/test/test_submodule.py index f63db1495..f706769c0 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -3,17 +3,17 @@ import contextlib import os -import shutil -import tempfile +import os.path as osp from pathlib import Path +import shutil import sys +import tempfile from unittest import mock, skipUnless import pytest import git from git.cmd import Git -from git.compat import is_win from git.config import GitConfigParser, cp from git.exc import ( GitCommandError, @@ -25,11 +25,8 @@ from git.objects.submodule.base import Submodule from git.objects.submodule.root import RootModule, RootUpdateProgress from git.repo.fun import find_submodule_git_dir, touch -from test.lib import TestBase, with_rw_repo -from test.lib import with_rw_directory -from git.util import HIDE_WINDOWS_KNOWN_ERRORS -from git.util import to_native_path_linux, join_path_native -import os.path as osp +from git.util import HIDE_WINDOWS_KNOWN_ERRORS, join_path_native, to_native_path_linux +from test.lib import TestBase, with_rw_directory, with_rw_repo @contextlib.contextmanager @@ -1040,7 +1037,7 @@ def test_branch_renames(self, rw_dir): assert sm_mod.commit() == sm_pfb.commit, "Now head should have been reset" assert sm_mod.head.ref.name == sm_pfb.name - @skipUnless(is_win, "Specifically for Windows.") + @skipUnless(os.name == "nt", "Specifically for Windows.") def test_to_relative_path_with_super_at_root_drive(self): class Repo: working_tree_dir = "D:\\" diff --git a/test/test_util.py b/test/test_util.py index d345247b1..4ef98fd73 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -20,7 +20,6 @@ import pytest from git.cmd import dashify -from git.compat import is_win from git.objects.util import ( altz_to_utctz_str, from_timestamp, @@ -364,7 +363,7 @@ def test_blocking_lock_file(self): self.assertRaises(IOError, wait_lock._obtain_lock) elapsed = time.time() - start extra_time = 0.02 - if is_win or sys.platform == "cygwin": + if os.name == "nt" or sys.platform == "cygwin": extra_time *= 6 # NOTE: Indeterministic failures without this... self.assertLess(elapsed, wait_time + extra_time) From 562bdee8110267f2861625f40aa403a72f000def Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 4 Nov 2023 17:33:58 -0400 Subject: [PATCH 0849/1790] Fix compat.is_darwin This fixes compat.is_darwin to use sys.platform instead of os.name so that it is True on macOS systems instead of always being False. The deprecation rationale in compat.py is accordingly adjusted. Together with c01fe76, this largely addresses #1731. --- git/compat.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/git/compat.py b/git/compat.py index 545bf65b4..629be075a 100644 --- a/git/compat.py +++ b/git/compat.py @@ -36,17 +36,14 @@ # DEPRECATED attributes providing shortcuts to operating system checks based on os.name. # -# - is_win and is_posix are deprecated because it is clearer, and helps avoid bugs, to -# write out the os.name checks explicitly. For example, is_win is False on Cygwin, but -# is often assumed to be True. +# These are deprecated because it is clearer, and helps avoid bugs, to write out the +# os.name or sys.platform checks explicitly, especially in cases where it matters which +# is used. For example, is_win is False on Cygwin, but is often assumed True. To detect +# Cygwin, use sys.platform == "cygwin". (Also, in the past, is_darwin was unreliable.) # -# - is_darwin is deprecated because it is always False on all systems, as os.name is -# never "darwin". For macOS, you can check for sys.platform == "darwin". (As on other -# Unix-like systems, os.name == "posix" on macOS. This is also the case on Cygwin.) -# -is_win: bool = os.name == "nt" +is_win = os.name == "nt" is_posix = os.name == "posix" -is_darwin = os.name == "darwin" +is_darwin = sys.platform == "darwin" defenc = sys.getfilesystemencoding() From 86453efa47c8a32b8b66446dac10686bb06f2e0a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 5 Nov 2023 14:53:17 +0100 Subject: [PATCH 0850/1790] Create FUNDING.json to support Drips --- FUNDING.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 FUNDING.json diff --git a/FUNDING.json b/FUNDING.json new file mode 100644 index 000000000..bf3faa662 --- /dev/null +++ b/FUNDING.json @@ -0,0 +1,7 @@ +{ + "drips": { + "ethereum": { + "ownedBy": "0xD0d4dCFc194ec24bCc777e635289e0b10E1a7b87" + } + } +} From aed3b59d50b7105debcb11d03c7abd8a1a4e5c96 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 6 Nov 2023 10:05:57 -0500 Subject: [PATCH 0851/1790] Revise and restore some module docstrings In 9ccd777, several module docstrings were converted into comments or outright deleted, apparently due to at least one of them (in git/index/fun.py) and possibly others having broken Sphinx autodoc. As discussed later in #1733, this no longer seems to be a problem, not even in git/index/fun.py where the comment included a note indicating it had. So this converts them back to docstrings, re-adding those that were removed in 9ccd777. This also revises them, partly to avoid making outdated claims, but mostly for style and clarity. It also revises some already-present module docstrings. Other than modules 9ccd777 affected, this does not add new docstrings to modules without them. One of the revisions made to some module docstrings here, to improve readability, is to remove language like "Module implementing" or "Module for", which was already used only in some places. However, this language is retained in the specific cases where it is clarifying, for example by avoiding documenting a module as if it the same thing as a single class implemented in it (which would be distracting because it would read as a mistake, and which could be confusing in cases where a developer uses an import with an "as" clause and needs to debug it because it was intended to be an "import" or "from" statement but was accidentally written as the other of those). --- git/config.py | 3 +-- git/exc.py | 2 +- git/index/base.py | 3 +++ git/index/fun.py | 3 +-- git/index/typ.py | 2 +- git/index/util.py | 2 +- git/objects/fun.py | 2 +- git/objects/tag.py | 2 +- git/objects/util.py | 2 +- git/refs/head.py | 5 +++++ git/refs/remote.py | 2 ++ git/repo/fun.py | 2 +- test/test_blob_filter.py | 2 +- test/test_exc.py | 1 - test/test_repo.py | 1 + 15 files changed, 21 insertions(+), 13 deletions(-) diff --git a/git/config.py b/git/config.py index 17a07fba9..983d71e86 100644 --- a/git/config.py +++ b/git/config.py @@ -3,8 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module containing module parser implementation able to properly read and write -configuration files.""" +"""Parser for reading and writing configuration files.""" import abc import configparser as cp diff --git a/git/exc.py b/git/exc.py index 35008c29a..85113bc44 100644 --- a/git/exc.py +++ b/git/exc.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module containing all exceptions thrown throughout the git package.""" +"""Exceptions thrown throughout the git package.""" __all__ = [ # Defined in gitdb.exc: diff --git a/git/index/base.py b/git/index/base.py index dbfa2c9bf..94437ac88 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -3,6 +3,9 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module containing IndexFile, an Index implementation facilitating all kinds of index +manipulations such as querying and merging.""" + from contextlib import ExitStack import datetime import glob diff --git a/git/index/fun.py b/git/index/fun.py index 45317429f..7b3b06269 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -1,8 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# Standalone functions to accompany the index implementation and make it more versatile. -# NOTE: Autodoc hates it if this is a docstring. +"""Standalone functions to accompany the index implementation and make it more versatile.""" from io import BytesIO import os diff --git a/git/index/typ.py b/git/index/typ.py index 7011fd03d..894e6f16d 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module with additional types used by the index.""" +"""Additional types used by the index.""" from binascii import b2a_hex from pathlib import Path diff --git a/git/index/util.py b/git/index/util.py index 5ee20a8ef..b1aaa58fd 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module containing index utilities.""" +"""Index utilities.""" from functools import wraps import os diff --git a/git/objects/fun.py b/git/objects/fun.py index 6d8a23d35..bf6bc21d6 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module with functions which are supposed to be as fast as possible.""" +"""Functions that are supposed to be as fast as possible.""" from stat import S_ISDIR diff --git a/git/objects/tag.py b/git/objects/tag.py index 500879d54..f7aaaf477 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module containing all Object-based types.""" +"""Object-based types.""" from . import base from .util import get_object_type_by_name, parse_actor_and_date diff --git a/git/objects/util.py b/git/objects/util.py index 9f42227d0..3021fec39 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module for general utility functions.""" +"""General utility functions.""" # flake8: noqa F401 diff --git a/git/refs/head.py b/git/refs/head.py index fba195aaa..ebc71eb96 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -1,6 +1,11 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +"""Some ref-based objects. + +Note the distinction between the :class:`HEAD` and :class:`Head` classes. +""" + from git.config import GitConfigParser, SectionConstraint from git.util import join_path from git.exc import GitCommandError diff --git a/git/refs/remote.py b/git/refs/remote.py index dd4117fa7..59d02a755 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -1,6 +1,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +"""Module implementing a remote object allowing easy access to git remotes.""" + import os from git.util import join_path diff --git a/git/repo/fun.py b/git/repo/fun.py index ee831332f..63bcfdfc7 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module with general repository-related functions.""" +"""General repository-related functions.""" from __future__ import annotations diff --git a/test/test_blob_filter.py b/test/test_blob_filter.py index 5cc6b48c9..a91f211bf 100644 --- a/test/test_blob_filter.py +++ b/test/test_blob_filter.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Test the blob filter.""" +"""Tests for the blob filter.""" from pathlib import Path from typing import Sequence, Tuple diff --git a/test/test_exc.py b/test/test_exc.py index 62bb4fb6e..cecd4f342 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -3,7 +3,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ - import re import ddt diff --git a/test/test_repo.py b/test/test_repo.py index e77bf2503..e39aee05e 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -2,6 +2,7 @@ # # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + import glob import io from io import BytesIO From 53354162f8e785ce39fb5dda85929fdc10e2b1b6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 04:36:43 -0400 Subject: [PATCH 0852/1790] Test that rmtree doesn't chmod outside the tree This adds a test in test_util that reveals the bug where git.util.rmtree will change the permissions on files outside the tree being removed, if the tree being removed contains a symlink to a file outside the tree, and the symlink is in a (sub)directory whose own permissions prevent the symlink itself from being removed. The new test failure shows how git.util.rmtree currently calls os.chmod in a way that dereferences symlinks, including those that point from inside the tree being deleted to outside it. Another similar demonstration is temporarily included in the perm.sh script. That script served as scratchwork for writing the unit test, and it can be removed soon (while keeping the test). --- perm.sh | 17 +++++++++++++++++ test/test_util.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100755 perm.sh diff --git a/perm.sh b/perm.sh new file mode 100755 index 000000000..419f0c1a8 --- /dev/null +++ b/perm.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +mkdir dir1 +touch dir1/file +chmod -w dir1/file +printf 'Permissions BEFORE rmtree call:\n' +ls -l dir1/file +printf '\n' + +mkdir dir2 +ln -s ../dir1/file dir2/symlink +chmod -w dir2 +python -c 'from git.util import rmtree; rmtree("dir2")' || true +printf '\nPermissions AFTER rmtree call:\n' +ls -l dir1/file diff --git a/test/test_util.py b/test/test_util.py index 07568c32a..148f852d5 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -105,6 +105,35 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): assert not td.exists() + @pytest.mark.skipif( + sys.platform == "cygwin", + reason="Cygwin can't set the permissions that make the test meaningful.", + ) + def test_avoids_changing_permissions_outside_tree(self, tmp_path: pathlib.Path): + # Automatically works on Windows, but on Unix requires either special handling + # or refraining from attempting to fix PermissionError by making chmod calls. + + dir1 = tmp_path / "dir1" + dir1.mkdir() + (dir1 / "file").write_bytes(b"") + (dir1 / "file").chmod(stat.S_IRUSR) + old_mode = (dir1 / "file").stat().st_mode + + dir2 = tmp_path / "dir2" + dir2.mkdir() + (dir2 / "symlink").symlink_to(dir1 / "file") + dir2.chmod(stat.S_IRUSR | stat.S_IXUSR) + + try: + rmtree(dir2) + except PermissionError: + pass # On Unix, dir2 is not writable, so dir2/symlink may not be deleted. + except SkipTest as ex: + self.fail(f"rmtree unexpectedly attempts skip: {ex!r}") + + new_mode = (dir1 / "file").stat().st_mode + assert old_mode == new_mode, f"Should stay {old_mode:#o}, became {new_mode:#o}." + @pytest.mark.skipif( sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", From fd78a53f70a70c2ce8d891f8eb0692f4ab787e25 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 30 Oct 2023 17:58:44 -0400 Subject: [PATCH 0853/1790] One approach to the symlink-following bug --- git/util.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 63619b87d..c0cd2f152 100644 --- a/git/util.py +++ b/git/util.py @@ -197,7 +197,7 @@ def patch_env(name: str, value: str) -> Generator[None, None, None]: os.environ[name] = old_value -def rmtree(path: PathLike) -> None: +def _rmtree_windows(path: PathLike) -> None: """Remove the given directory tree recursively. :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files @@ -225,6 +225,17 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: shutil.rmtree(path, onerror=handler) +def _rmtree_posix(path: PathLike) -> None: + """Remove the given directory tree recursively.""" + return shutil.rmtree(path) + + +if os.name == "nt": + rmtree = _rmtree_windows +else: + rmtree = _rmtree_posix + + def rmfile(path: PathLike) -> None: """Ensure file deleted also on *Windows* where read-only files need special treatment.""" if osp.isfile(path): From 6de8e676c55170f073b64471e78b606c9c01b757 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Nov 2023 16:43:48 -0500 Subject: [PATCH 0854/1790] Refactor current fix for symlink-following bug This keeps the same approach, but expresses it better. This does not update the tests (yet), so one test is still failing. --- git/util.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/git/util.py b/git/util.py index c0cd2f152..01cdcf773 100644 --- a/git/util.py +++ b/git/util.py @@ -197,7 +197,7 @@ def patch_env(name: str, value: str) -> Generator[None, None, None]: os.environ[name] = old_value -def _rmtree_windows(path: PathLike) -> None: +def rmtree(path: PathLike) -> None: """Remove the given directory tree recursively. :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files @@ -219,23 +219,14 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex raise - if sys.version_info >= (3, 12): + if os.name != "nt": + shutil.rmtree(path) + elif sys.version_info >= (3, 12): shutil.rmtree(path, onexc=handler) else: shutil.rmtree(path, onerror=handler) -def _rmtree_posix(path: PathLike) -> None: - """Remove the given directory tree recursively.""" - return shutil.rmtree(path) - - -if os.name == "nt": - rmtree = _rmtree_windows -else: - rmtree = _rmtree_posix - - def rmfile(path: PathLike) -> None: """Ensure file deleted also on *Windows* where read-only files need special treatment.""" if osp.isfile(path): From e8cfae89657b941ffd824afc9c11891d79952624 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Nov 2023 18:26:02 -0500 Subject: [PATCH 0855/1790] Test that PermissionError is only wrapped on Windows This changes tests in test_util to verify the opposite behavior from what was enforced before, in the unusual case (that hopefully never happens outside of monkey-patching in test_util.py itself) where the system is not Windows but HIDE_WINDOWS_KNOWN_ERRORS is set to True. The same-named environment variable will not, and never has, set HIDE_WINDOWS_KNOWN_ERRORS to True on non-Windows systems, but it is possible to set it to True directly. Since it is named as a constant and no documentation has ever suggested changing its value directly, nor otherwise attempting to use it outside Windows, it shouldn't matter what happens in this unusual case. But asserting that wrapping never occurs in this combination of circumstances is what makes the most sense in light of the recent change to pass no callback to shutil.rmtree on non-Windows systems. --- test/test_util.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 148f852d5..40dd4f1fd 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -135,11 +135,11 @@ def test_avoids_changing_permissions_outside_tree(self, tmp_path: pathlib.Path): assert old_mode == new_mode, f"Should stay {old_mode:#o}, became {new_mode:#o}." @pytest.mark.skipif( - sys.platform == "cygwin", - reason="Cygwin can't set the permissions that make the test meaningful.", + os.name != "nt", + reason="PermissionError is only ever wrapped on Windows", ) def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): - """rmtree wraps PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is true.""" # Access the module through sys.modules so it is unambiguous which module's # attribute we patch: the original git.util, not git.index.util even though # git.index.util "replaces" git.util and is what "import git.util" gives us. @@ -157,10 +157,17 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", ) - def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir): - """rmtree does not wrap PermissionError when HIDE_WINDOWS_KNOWN_ERRORS is false.""" + @pytest.mark.parametrize( + "hide_windows_known_errors", + [ + pytest.param(False), + pytest.param(True, marks=pytest.mark.skipif(os.name == "nt", reason="We would wrap on Windows")), + ], + ) + def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir, hide_windows_known_errors): + """rmtree does not wrap PermissionError on non-Windows systems or when HIDE_WINDOWS_KNOWN_ERRORS is false.""" # See comments in test_wraps_perm_error_if_enabled for details about patching. - mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", False) + mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) mocker.patch.object(os, "chmod") mocker.patch.object(pathlib.Path, "chmod") From 8a22352407638923b9b89be66a30a44b0bb50690 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Nov 2023 18:47:19 -0500 Subject: [PATCH 0856/1790] Extract some shared patching code to a helper Because of how it is parameterized, but only in some of the test cases that use it, making this a pytest fixture would be cumbersome without further refactoring, so this is a helper method instead. (It may be feasible to further refactor the test suite to improve this more.) This also removes a type annotation from a test method. It may eventually be helpful to add annotations, in which case that would come back along with others (if that test still exists in this form), but it's the only test with such an annotation in this test class, and the annotation had been added accidentally. --- test/test_util.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 40dd4f1fd..e5a5a0557 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -109,7 +109,7 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", ) - def test_avoids_changing_permissions_outside_tree(self, tmp_path: pathlib.Path): + def test_avoids_changing_permissions_outside_tree(self, tmp_path): # Automatically works on Windows, but on Unix requires either special handling # or refraining from attempting to fix PermissionError by making chmod calls. @@ -134,22 +134,24 @@ def test_avoids_changing_permissions_outside_tree(self, tmp_path: pathlib.Path): new_mode = (dir1 / "file").stat().st_mode assert old_mode == new_mode, f"Should stay {old_mode:#o}, became {new_mode:#o}." - @pytest.mark.skipif( - os.name != "nt", - reason="PermissionError is only ever wrapped on Windows", - ) - def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): - """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + def _patch_for_wrapping_test(self, mocker, hide_windows_known_errors): # Access the module through sys.modules so it is unambiguous which module's # attribute we patch: the original git.util, not git.index.util even though # git.index.util "replaces" git.util and is what "import git.util" gives us. - mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", True) + mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) - # Disable common chmod functions so the callback can never fix the problem. + # Disable common chmod functions so the callback can't fix a PermissionError. mocker.patch.object(os, "chmod") mocker.patch.object(pathlib.Path, "chmod") - # Now we can see how an intractable PermissionError is treated. + @pytest.mark.skipif( + os.name != "nt", + reason="PermissionError is only ever wrapped on Windows", + ) + def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): + """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + self._patch_for_wrapping_test(mocker, True) + with pytest.raises(SkipTest): rmtree(permission_error_tmpdir) @@ -166,10 +168,7 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): ) def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir, hide_windows_known_errors): """rmtree does not wrap PermissionError on non-Windows systems or when HIDE_WINDOWS_KNOWN_ERRORS is false.""" - # See comments in test_wraps_perm_error_if_enabled for details about patching. - mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) - mocker.patch.object(os, "chmod") - mocker.patch.object(pathlib.Path, "chmod") + self._patch_for_wrapping_test(mocker, hide_windows_known_errors) with pytest.raises(PermissionError): try: @@ -180,11 +179,7 @@ def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_ @pytest.mark.parametrize("hide_windows_known_errors", [False, True]) def test_does_not_wrap_other_errors(self, tmp_path, mocker, hide_windows_known_errors): file_not_found_tmpdir = tmp_path / "testdir" # It is deliberately never created. - - # See comments in test_wraps_perm_error_if_enabled for details about patching. - mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) - mocker.patch.object(os, "chmod") - mocker.patch.object(pathlib.Path, "chmod") + self._patch_for_wrapping_test(mocker, hide_windows_known_errors) with pytest.raises(FileNotFoundError): try: From b226f3dd04c16d8fc4ef4044ad81bb72be8683d1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Nov 2023 19:28:03 -0500 Subject: [PATCH 0857/1790] Remove demonstration script This removes the perm.sh script that demonstrated the cross-tree permission change bug. Tests for test_util test for this (in case of a regression) and the bug is fixed, so the script, which served to inform the writing of the corresponding unit test, is no longer needed. --- perm.sh | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100755 perm.sh diff --git a/perm.sh b/perm.sh deleted file mode 100755 index 419f0c1a8..000000000 --- a/perm.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -set -e - -mkdir dir1 -touch dir1/file -chmod -w dir1/file -printf 'Permissions BEFORE rmtree call:\n' -ls -l dir1/file -printf '\n' - -mkdir dir2 -ln -s ../dir1/file dir2/symlink -chmod -w dir2 -python -c 'from git.util import rmtree; rmtree("dir2")' || true -printf '\nPermissions AFTER rmtree call:\n' -ls -l dir1/file From 359116b3b9b7011eb7893fa584a43522f1b9bb57 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 13 Nov 2023 23:48:13 -0500 Subject: [PATCH 0858/1790] List all non-passing tests in test summaries This changes the default in pyproject.toml so that pytest lists a line for each non-passing test at the end of a run, showing the test name and, where available, condensed information about the status, such as the "reason" argument for an xfailing or skipped test. Previously only failed and errored tests were listed in the summary. Now skipped, xfailed, and xpassed tests are listed too. The benefit is in keeping track of the status of tests. Although showing the full failure output with stack trace and relevant code under test would be too distracting for tests marked xfail, it is valuable to not merely run those tests but be able to see a line showing their names and statuses. Likewise, a number of tests are currently marked skipped, and while some of them are skipped on a particular platform because they don't make sense to run on that platform, a number of others are skipped by raising SkipTest in response to a failure condition on Windows. (Those consist mostly of the tests skipped as a result of code discussed in #790.) This also has the more specific benefit of making it easier to mark tests as xfail in order to add CI jobs for native Windows, and more importantly to allow information about their status to later be used to understand and fix bugs on Windows.) --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eae5943ea..7109389d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -addopts = "--cov=git --cov-report=term --disable-warnings" +addopts = "--cov=git --cov-report=term --disable-warnings -ra" filterwarnings = "ignore::DeprecationWarning" python_files = "test_*.py" tmp_path_retention_policy = "failed" @@ -14,8 +14,8 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # --cov-report html:path # html file at path # --maxfail # number of errors before giving up # -disable-warnings # Disable pytest warnings (not codebase warnings) -# -rf # increased reporting of failures -# -rE # increased reporting of errors +# -rfE # default test summary: list fail and error +# -ra # test summary: list all non-passing (fail, error, skip, xfail, xpass) # --ignore-glob=**/gitdb/* # ignore glob paths # filterwarnings ignore::WarningType # ignores those warnings From 2fd79f41fe1304be7bcaebec84394ab9e45961c5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Oct 2023 02:35:44 -0400 Subject: [PATCH 0859/1790] Add native Windows test jobs to CI matrix This expands the CI test matrix in the main testing workflow to test on both Ubuntu and Windows, instead of just Ubuntu. It does not attempt to merge in the Cygwin workflow at this time, which may or may not end up being useful to do in the future. The new Windows test jobs all fail currently: the runs fail as a result of various tests being consistently unable to pass but not yet being marked xfail (or skip). It should be feasible to mark these xfail with informative messages, but this commit doesn't include that. --- .github/workflows/pythonpackage.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 35fecf16c..95b12004c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -10,18 +10,19 @@ permissions: jobs: build: - runs-on: ubuntu-latest - strategy: fail-fast: false matrix: + os: ["ubuntu-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false + runs-on: ${{ matrix.os }} + defaults: run: - shell: /bin/bash --noprofile --norc -exo pipefail {0} + shell: bash --noprofile --norc -exo pipefail {0} steps: - uses: actions/checkout@v4 From 6e477e3a06e4b11d43ea118a9f18cae03fa211fd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 14 Nov 2023 00:41:57 -0500 Subject: [PATCH 0860/1790] Add xfail marks for IndexFile.from_tree failures 8 of the tests that fail on native Windows systems fail due to IndexFile.from_tree being broken on Windows, causing #1630. This commit marks those tests as xfail. This is part, though not all, of the changes to get CI test jobs for native Windows that are passing, to guard against new regressions and to allow the code and tests to be gradually fixed (see discussion in #1654). When fixing the bug, this commit can be reverted. --- test/test_docs.py | 11 +++++++++-- test/test_fun.py | 17 ++++++++++++++--- test/test_index.py | 40 ++++++++++++++++++++++++++++++++++++++++ test/test_refs.py | 11 +++++++++++ 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/test/test_docs.py b/test/test_docs.py index 2f4b2e8d8..2ff1c794a 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -8,11 +8,10 @@ import pytest +from git.exc import GitCommandError from test.lib import TestBase from test.lib.helper import with_rw_directory -import os.path - class Tutorials(TestBase): def tearDown(self): @@ -207,6 +206,14 @@ def update(self, op_code, cur_count, max_count=None, message=""): assert sm.module_exists() # The submodule's working tree was checked out by update. # ![14-test_init_repo_object] + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_directory def test_references_and_objects(self, rw_dir): # [1-test_references_and_objects] diff --git a/test/test_fun.py b/test/test_fun.py index 566bc9aae..8ea5b7e46 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -3,10 +3,13 @@ from io import BytesIO from stat import S_IFDIR, S_IFREG, S_IFLNK, S_IXUSR -from os import stat +import os import os.path as osp +import pytest + from git import Git +from git.exc import GitCommandError from git.index import IndexFile from git.index.fun import ( aggressive_tree_merge, @@ -34,6 +37,14 @@ def _assert_index_entries(self, entries, trees): assert (entry.path, entry.stage) in index.entries # END assert entry matches fully + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) def test_aggressive_tree_merge(self): # Head tree with additions, removals and modification compared to its predecessor. odb = self.rorepo.odb @@ -291,12 +302,12 @@ def test_linked_worktree_traversal(self, rw_dir): rw_master.git.worktree("add", worktree_path, branch.name) dotgit = osp.join(worktree_path, ".git") - statbuf = stat(dotgit) + statbuf = os.stat(dotgit) self.assertTrue(statbuf.st_mode & S_IFREG) gitdir = find_worktree_git_dir(dotgit) self.assertIsNotNone(gitdir) - statbuf = stat(gitdir) + statbuf = os.stat(gitdir) self.assertTrue(statbuf.st_mode & S_IFDIR) def test_tree_entries_from_data_with_failing_name_decode_py3(self): diff --git a/test/test_index.py b/test/test_index.py index 3d1042be8..9fe35d42b 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -179,6 +179,14 @@ def add_bad_blob(): except Exception as ex: assert "index.lock' could not be obtained" not in str(ex) + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_repo("0.1.6") def test_index_file_from_tree(self, rw_repo): common_ancestor_sha = "5117c9c8a4d3af19a9958677e45cda9269de1541" @@ -229,6 +237,14 @@ def test_index_file_from_tree(self, rw_repo): # END for each blob self.assertEqual(num_blobs, len(three_way_index.entries)) + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_repo("0.1.6") def test_index_merge_tree(self, rw_repo): # A bit out of place, but we need a different repo for this: @@ -291,6 +307,14 @@ def test_index_merge_tree(self, rw_repo): self.assertEqual(len(unmerged_blobs), 1) self.assertEqual(list(unmerged_blobs.keys())[0], manifest_key[0]) + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_repo("0.1.6") def test_index_file_diffing(self, rw_repo): # Default Index instance points to our index. @@ -425,6 +449,14 @@ def _count_existing(self, repo, files): # END num existing helper + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): index = rw_repo.index @@ -778,6 +810,14 @@ def make_paths(): for absfile in absfiles: assert osp.isfile(absfile) + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_repo("HEAD") def test_compare_write_tree(self, rw_repo): """Test writing all trees, comparing them for equality.""" diff --git a/test/test_refs.py b/test/test_refs.py index 6ee385007..a1573c11b 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -4,8 +4,11 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from itertools import chain +import os from pathlib import Path +import pytest + from git import ( Reference, Head, @@ -215,6 +218,14 @@ def test_head_checkout_detached_head(self, rw_repo): assert isinstance(res, SymbolicReference) assert res.name == "HEAD" + @pytest.mark.xfail( + os.name == "nt", + reason=( + "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" + "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." + ), + raises=GitCommandError, + ) @with_rw_repo("0.1.6") def test_head_reset(self, rw_repo): cur_head = rw_repo.head From cd9d7a9d558273a1f82527890a1d69529a3ef1d5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 09:55:06 -0500 Subject: [PATCH 0861/1790] Mark test_clone_command_injection xfail on Windows This other GitCommandError on Windows is not related to IndexFile.from_tree whose 8 related failing tests were marked xfail in the preceding commit. Also, test_clone_command_injection should not be confused with test_clone_from_command_injection, which passes on all platforms. The problem here appears to be that, on Windows, the path of the directory GitPython is intended to clone to -- when the possible security vulnerability this test checks for is absent -- is not valid. This suggests the bug may only be in the test and that the code under test may be working on Windows. But the test does not establish that, for which it would need to test with a payload clearly capable of creating the file unexpected_path refers to when run on its own. This doesn't currently seem to be reported as a bug, but some general context about the implementation can be examined in #1518 where it was introduced, and #1531 where it was modified. --- test/test_repo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_repo.py b/test/test_repo.py index e39aee05e..f853e2b36 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1365,6 +1365,11 @@ def test_do_not_strip_newline_in_stdout(self, rw_dir): r.git.commit(message="init") self.assertEqual(r.git.show("HEAD:hello.txt", strip_newline_in_stdout=False), "hello\n") + @pytest.mark.xfail( + os.name == "nt", + reason=R"fatal: could not create leading directories of '--upload-pack=touch C:\Users\ek\AppData\Local\Temp\tmpnantqizc\pwn': Invalid argument", # noqa: E501 + raises=GitCommandError, + ) @with_rw_repo("HEAD") def test_clone_command_injection(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: From f72e2821a3073c4e87915fe7e721512e16e4fa74 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 12:19:10 -0500 Subject: [PATCH 0862/1790] Mark test_diff_submodule xfail on Windows Everything attempted in the test case method is actually working and passes, but when the tearDown method calls shutil.rmtree, it fails with "Access is denied" and raises PermissionError. The reason and exception are accordingly noted in the xfail decoration. While adding a pytest import to apply the pytest xfail mark, I also improved grouping/sorting of other imports in the test_diff module. --- test/test_diff.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/test_diff.py b/test/test_diff.py index 1678e737d..1d138a086 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -3,26 +3,17 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import ddt +import os +import os.path as osp import shutil import tempfile -from git import ( - Repo, - GitCommandError, - Diff, - DiffIndex, - NULL_TREE, - Submodule, -) -from git.cmd import Git -from test.lib import ( - TestBase, - StringProcessAdapter, - fixture, -) -from test.lib import with_rw_directory -import os.path as osp +import ddt +import pytest + +from git import NULL_TREE, Diff, DiffIndex, GitCommandError, Repo, Submodule +from git.cmd import Git +from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory def to_raw(input): @@ -318,6 +309,11 @@ def test_diff_with_spaces(self): self.assertIsNone(diff_index[0].a_path, repr(diff_index[0].a_path)) self.assertEqual(diff_index[0].b_path, "file with spaces", repr(diff_index[0].b_path)) + @pytest.mark.xfail( + os.name == "nt", + reason='"Access is denied" when tearDown calls shutil.rmtree', + raises=PermissionError, + ) def test_diff_submodule(self): """Test that diff is able to correctly diff commits that cover submodule changes""" # Init a temp git repo that will be referenced as a submodule. From 42a3d74f529f34382b205c0ada46fb18df80c034 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 14:14:44 -0500 Subject: [PATCH 0863/1790] Mark TestSubmodule.test_rename xfail on Windows The test fails when attempting to evaluate the expression: sm.move(new_path).name as part of the assertion: assert sm.move(new_path).name == new_path But it is the call to sm.move that fails, not the assertion itself. The condition is never evaluated, because the subexpression raises PermissionError. Details are in the xfail reason string. This test_submodule.TestSubmodule.test_rename method should not be confused with test_config.TestBase.test_rename, which passes on all platforms. On CI this has XPASS status on Python 3.7 and possibly some other versions. This should be investigated and, if possible, the xfail markings should be made precise. (When working on this change before a rebase, I had thought only 3.7 xpassed, and that the XPASS was only on CI, never locally. However, I am unable to reproduce that or find CI logs of it, so some of these observations may have been mistaken and/or or specific to a narrow environment.) --- test/test_submodule.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index 5be1a1077..68dbcc491 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -949,6 +949,17 @@ def test_remove_norefs(self, rwdir): sm.remove() assert not sm.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=( + "The sm.move call fails. Submodule.move calls os.renames, which raises:\n" + "PermissionError: [WinError 32] " + "The process cannot access the file because it is being used by another process: " + R"'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\mymodules\myname' " + R"-> 'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\renamed\myname'" + ), + raises=PermissionError, + ) @with_rw_directory def test_rename(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) From 4abab92cb0f4caf569cc4bd9a8084994a80733ce Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 14:54:53 -0500 Subject: [PATCH 0864/1790] Mark test_conditional_includes_from_git_dir xfail on Windows As noted, the second of the config._included_paths() assertions fails, which is in the "Ensure that config is included if path is matching git_dir" sub-case. It is returning 0 on Windows. THe GitConfigParser._has_includes function returns the expression: self._merge_includes and len(self._included_paths()) Since _merge_includes is a bool, it appears the first branch of the "and" is True, but then _included_paths returns an empty list. --- test/test_config.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/test_config.py b/test/test_config.py index 0e1bba08a..c1b26c91f 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -6,19 +6,15 @@ import glob import io import os +import os.path as osp from unittest import mock +import pytest + from git import GitConfigParser from git.config import _OMD, cp -from test.lib import ( - TestCase, - fixture_path, - SkipTest, -) -from test.lib import with_rw_directory - -import os.path as osp from git.util import rmfile +from test.lib import SkipTest, TestCase, fixture_path, with_rw_directory _tc_lock_fpaths = osp.join(osp.dirname(__file__), "fixtures/*.lock") @@ -239,6 +235,11 @@ def check_test_value(cr, value): with GitConfigParser(fpa, read_only=True) as cr: check_test_value(cr, tv) + @pytest.mark.xfail( + os.name == "nt", + reason='Second config._has_includes() assertion fails (for "config is included if path is matching git_dir")', + raises=AssertionError, + ) @with_rw_directory def test_conditional_includes_from_git_dir(self, rw_dir): # Initiate repository path. From 799c8536800e75d7ddaef2b588754234db363245 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 19:57:15 -0500 Subject: [PATCH 0865/1790] Improve ordering/grouping of a few imports In modules soon to be modified, so if subsequent commits are later reverted, these import tweaks are not automatically undone. --- test/test_remote.py | 27 ++++++++++++++------------- test/test_repo.py | 25 ++++++++++++------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/test/test_remote.py b/test/test_remote.py index 12de18476..711c7b1bc 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -3,36 +3,37 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os.path as osp +from pathlib import Path import random import tempfile -import pytest from unittest import skipIf +import pytest + from git import ( - RemoteProgress, + Commit, FetchInfo, - Reference, - SymbolicReference, + GitCommandError, Head, - Commit, PushInfo, + Reference, + Remote, + RemoteProgress, RemoteReference, + SymbolicReference, TagReference, - Remote, - GitCommandError, ) from git.cmd import Git -from pathlib import Path from git.exc import UnsafeOptionError, UnsafeProtocolError +from git.util import rmtree, HIDE_WINDOWS_FREEZE_ERRORS, IterableList from test.lib import ( + GIT_DAEMON_PORT, TestBase, - with_rw_repo, - with_rw_and_rw_remote_repo, fixture, - GIT_DAEMON_PORT, + with_rw_and_rw_remote_repo, + with_rw_repo, ) -from git.util import rmtree, HIDE_WINDOWS_FREEZE_ERRORS, IterableList -import os.path as osp # Make sure we have repeatable results. diff --git a/test/test_repo.py b/test/test_repo.py index f853e2b36..3cde25184 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -8,6 +8,7 @@ from io import BytesIO import itertools import os +import os.path as osp import pathlib import pickle import sys @@ -17,22 +18,22 @@ import pytest from git import ( + BadName, + Commit, + Git, + GitCmdObjectDB, + GitCommandError, + GitDB, + Head, + IndexFile, InvalidGitRepositoryError, - Repo, NoSuchPathError, - Head, - Commit, Object, - Tree, - IndexFile, - Git, Reference, - GitDB, - Submodule, - GitCmdObjectDB, Remote, - BadName, - GitCommandError, + Repo, + Submodule, + Tree, ) from git.exc import ( BadObject, @@ -43,8 +44,6 @@ from git.util import bin_to_hex, cygpath, join_path_native, rmfile, rmtree from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo -import os.path as osp - def iter_flatten(lol): for items in lol: From b284ad70291d24024b840fa659f54276cf9ceaa5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 20:17:51 -0500 Subject: [PATCH 0866/1790] Mark test_create_remote_unsafe_url_allowed xfail on Windows The test mostly works on Windows, but it fails because the tmp_file path is expected to appear in remote_url with backslash separators, but (forward) slashes appear instead. --- test/test_remote.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_remote.py b/test/test_remote.py index 711c7b1bc..f9f35e5d8 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os import os.path as osp from pathlib import Path import random @@ -767,6 +768,11 @@ def test_create_remote_unsafe_url(self, rw_repo): Remote.create(rw_repo, "origin", url) assert not tmp_file.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=R"Multiple '\' instead of '/' in remote.url make it differ from expected value", + raises=AssertionError, + ) @with_rw_repo("HEAD") def test_create_remote_unsafe_url_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: From 61d1fba6dd318e7f68b93b0e8b46050db0d03fa8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 15 Nov 2023 21:04:04 -0500 Subject: [PATCH 0867/1790] Mark unsafe-options "allowed" tests xfail on Windows The tests of unsafe options are among those introduced originally in #1521. They are regression tests for #1515 (CVE-2022-24439). The unsafe options tests are paired: a test for the usual, default behavior of forbidding the option, and a test for the behavior when the option is explicitly allowed. In each such pair, both tests use a payload that is intended to produce the side effect of a file of a specific name being created in a temporary directory. All the tests work on Unix-like systems. On Windows, the tests of the *allowed* cases are broken, and this commit marks them xfail. However, this has implications for the tests of the default, secure behavior, because until the "allowed" versions work on Windows, it will be unclear if either are using a payload that is effective and that corresponds to the way its effect is examined. What *seems* to happen is this: The "\" characters in the path are treated as shell escape characters rather than literally, with the effect of disappearing in most paths since most letters lack special meaning when escaped. Also, "touch" is not a native Windows command, and the "touch" command provided by Git for Windows is linked against MSYS2 libraries, causing it to map (some?) occurrences of ":" in filenames to a separate code point in the Private Use Area of the Basic Multilingual Plane. The result is a path with no directory separators or drive letter. It denotes a file of an unintended name in the current directory, which is never the intended location. The current directory depends on GitPython implementation details, but at present it's the top-level directory of the rw_repo working tree. A new unstaged file, named like "C\357\200\272UsersekAppDataLocalTemptmpc7x4xik5pwn", can be observed there (this is how "git status" will format the name). Fortunately, this and all related tests are working on other OSes, and the affected code under test does not appear highly dependent on OS. So the fix is *probably* fully working on Windows as well. --- test/test_remote.py | 27 +++++++++++++++++++++++++++ test/test_repo.py | 18 ++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/test/test_remote.py b/test/test_remote.py index f9f35e5d8..080718a1c 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -831,6 +831,15 @@ def test_fetch_unsafe_options(self, rw_repo): remote.fetch(**unsafe_option) assert not tmp_file.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_fetch_unsafe_options must be adjusted in the " + "same way. Until then, test_fetch_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) @with_rw_repo("HEAD") def test_fetch_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: @@ -890,6 +899,15 @@ def test_pull_unsafe_options(self, rw_repo): remote.pull(**unsafe_option) assert not tmp_file.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_pull_unsafe_options must be adjusted in the " + "same way. Until then, test_pull_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) @with_rw_repo("HEAD") def test_pull_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: @@ -955,6 +973,15 @@ def test_push_unsafe_options(self, rw_repo): remote.push(**unsafe_option) assert not tmp_file.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_push_unsafe_options must be adjusted in the " + "same way. Until then, test_push_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) @with_rw_repo("HEAD") def test_push_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: diff --git a/test/test_repo.py b/test/test_repo.py index 3cde25184..033deca1e 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -294,6 +294,15 @@ def test_clone_unsafe_options(self, rw_repo): rw_repo.clone(tmp_dir, **unsafe_option) assert not tmp_file.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) @with_rw_repo("HEAD") def test_clone_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: @@ -364,6 +373,15 @@ def test_clone_from_unsafe_options(self, rw_repo): Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) assert not tmp_file.exists() + @pytest.mark.xfail( + os.name == "nt", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_from_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) @with_rw_repo("HEAD") def test_clone_from_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: From ad07ecb0151e64f3fa774099d8c69232df6bda23 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 22 Nov 2023 11:00:54 -0500 Subject: [PATCH 0868/1790] Show PATH on CI Along with the version and platform information that is already shown. This is to make debugging easier. --- .github/workflows/cygwin-test.yml | 1 + .github/workflows/pythonpackage.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 96728e51a..36329f93f 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -73,6 +73,7 @@ jobs: python -c 'import sys; print(sys.platform)' python -c 'import os; print(os.name)' python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. + printenv PATH | tr ':' '\n' - name: Test with pytest run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 95b12004c..890681b7d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -65,6 +65,7 @@ jobs: python -c 'import sys; print(sys.platform)' python -c 'import os; print(os.name)' python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. + printenv PATH | tr ':' '\n' - name: Check types with mypy run: | From 2784e403d6fd812650f29f1842db1e67344170a3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 22 Nov 2023 12:25:22 -0500 Subject: [PATCH 0869/1790] Show bash and other WSL-relevant info but not PATH Only on native Windows systems (since this is for debugging WSL). The Windows CI runners have WSL itself installed but no WSL systems installed. So some of these commands may fail if they call the bash.exe in the System32 directory, but this should still be illuminating. Results after a WSL system is set up will likely be more important, but doing this first allows for comparison. The reason PATH directories are no longer listed is to decrease noise. --- .github/workflows/cygwin-test.yml | 1 - .github/workflows/pythonpackage.yml | 22 +++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 36329f93f..96728e51a 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -73,7 +73,6 @@ jobs: python -c 'import sys; print(sys.platform)' python -c 'import os; print(os.name)' python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. - printenv PATH | tr ':' '\n' - name: Test with pytest run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 890681b7d..d21ea3949 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -65,7 +65,27 @@ jobs: python -c 'import sys; print(sys.platform)' python -c 'import os; print(os.name)' python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. - printenv PATH | tr ':' '\n' + + # For debugging hook tests on native Windows systems that may have WSL. + - name: Show where bash.exe may be found + if: startsWith(matrix.os, 'windows') + run: | + set +e + type -a bash.exe + python -c 'import shutil; print(shutil.which("bash.exe"))' + bash.exe --version + python -c 'import subprocess; p = subprocess.run(["bash.exe", "--version"]); print(f"result: {p!r}")' + bash.exe -c 'echo "$BASH"' + python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", """echo "$BASH" """]); print(f"result: {p!r}")' + bash.exe -c 'echo "$BASH_VERSION"' + python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", """echo "$BASH_VERSION" """]); print(f"result: {p!r}")' + bash.exe -c 'printenv WSL_DISTRO_NAME' + python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", "printenv WSL_DISTRO_NAME"]); print(f"result: {p!r}")' + bash.exe -c 'ls -l /proc/sys/fs/binfmt_misc/WSLInterop' + python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", "ls -l /proc/sys/fs/binfmt_misc/WSLInterop"]); print(f"result: {p!r}")' + bash.exe -c 'uname -a' + python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", "uname -a"]); print(f"result: {p!r}")' + continue-on-error: true - name: Check types with mypy run: | From 9717b8d847b514f761ecaa423226054c73f2a325 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 17 Nov 2023 11:22:12 -0500 Subject: [PATCH 0870/1790] Install WSL system on CI for hook tests This makes three of the four hook-related tests pass instead of failing, and changes the way the fourth fails. GitPython uses bash.exe to run hooks that are (or appear to be) shell scripts. On many Windows systems, this is the bash.exe in the system32 directory, which delegates to bash in a WSL system if at least one such system is installed (for the current user), and gives an error otherwise. It may be a bug that GitPython ends up using this bash.exe when WSL is installed but no WSL systems exist, since that is actually a fairly common situation. One place that happened was on the GitHub Actions runners used for Windows jobs. Those runners have WSL available, and are capable of running WSL 1 systems (not currently WSL 2 systems), but no WSL systems were actually installed. This commit fixes that cause of failure, for all four tests it happened in, by setting up a Debian WSL system on the test runner. (This increases the amount of time it takes Windows jobs to run, but that might be possible to improve on.) Three of those four tests now pass, while the other fails for another reason. The newly passing tests are: - test/test_index.py::TestIndex::test_commit_msg_hook_fail - test/test_index.py::TestIndex::test_pre_commit_hook_fail - test/test_index.py::TestIndex::test_pre_commit_hook_success The test that still fails, but differently, is: - test/test_index.py::TestIndex::test_commit_msg_hook_success I had previously found that test to fail on a local WSL 2 system, and attempted to add a suitable xfail marking in 881456b (#1679). But the condition I wrote there *appears* to have a bug related to the different orders in which subproces.Popen and shutil.which find executables, causing it not always to detect when the WSL-related bash.exe is the one the Popen call in git.index.fun.run_commit_hook will use. --- .github/workflows/pythonpackage.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d21ea3949..c2bbc1bbb 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,6 +35,12 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} + - name: Set up WSL + if: startsWith(matrix.os, 'windows') + uses: Vampire/setup-wsl@v2.0.2 + with: + distribution: Debian + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh From 5d113942786b2cc86f5fd7bb228f9f75c8c78beb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 23 Nov 2023 18:46:00 -0500 Subject: [PATCH 0871/1790] Fix and expand bash.exe xfail marks on hook tests 881456b (#1679) expanded the use of shutil.which in test_index.py to attempt to mark test_commit_msg_hook_success xfail when bash.exe is a WSL bash wrapper in System32 (because that test currently is not passing when the hook is run via bash in a WSL system, which the WSL bash.exe wraps). But this was not reliable, due to significant differences between shell and non-shell search behavior for executable commands on Windows. As the new docstring notes, lookup due to Popen generally checks System32 before going through directories in PATH, as this is how CreateProcessW behaves. - https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw - https://github.com/python/cpython/issues/91558#issuecomment-1100942950 The technique I had used in 881456b also had the shortcoming of assuming that a bash.exe in System32 was the WSL wrapper. But such a file may exist on some systems without WSL, and be a bash interpreter unrelated to WSL that may be able to run hooks. In addition, one common situation, which was the case on CI before a step to install a WSL distribution was added, is that WSL is present but no WSL distributions are installed. In that situation bash.exe is found in System32, but it can't be used to run any hooks, because there's no actual system with a bash in it to wrap. This was not covered before. Unlike some conditions that prevent a WSL system from being used, such as resource exhaustion blocking it from being started, the absence of a WSL system should probably not fail the pytest run, for the same reason as we are trying not to have the complete *absence* of bash.exe fail the pytest run. Both conditions should be xfail. Fortunately, the error message when no distribution exists is distinctive and can be checked for. There is probably no correct and reasonable way to check LBYL-style which executable file bash.exe resolves to by using shutil.which, due to shutil.which and subprocess.Popen's differing seach orders and other subtleties. So this adds code to do it EAFP-style using subprocess.run (which itself uses Popen, so giving the same CreateProcessW behavior). It tries to run a command with bash.exe whose output pretty reliably shows if the system is WSL or not. We may fail to get to the point of running that command at all, if bash.exe is not usable, in which case the failure's details tell us if bash.exe is absent (xfail), present as the WSL wrapper with no WSL systems (xfail), or has some other error (not xfail, so the user can become aware of and proably fix the problem). If we do get to that point, then a file that is almost always present on both WSL 1 and WSL 2 systems and almost always absent on any other system is checked for, to distinguish whether the working bash shell operates in a WSL system, or outside any such system as e.g. Git Bash does. See https://superuser.com/a/1749811 on various techniques for checking for WSL, including the /proc/sys/fs/binfmt_misc/WSLInterop technique used here (which seems overall may be the most reliable). Although the Windows CI runners have Git Bash, and this is even the bash.exe that appears first in PATH (giving rise to the problem with shutil.which detailed above), it would be a bit awkward to test the behavior when Git Bash is actually used to run hooks on CI, because of how Popen selects the System32 bash.exe first, whether or not any WSL distribution is present. However, local testing shows that when Git Bash's bash.exe is selected, all four hook tests in the module pass, both before and after this change, and furthermore that bash.exe is correctly detected as "native", continuing to avoid an erroneous xfail mark in that case. --- test/test_index.py | 126 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 13 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 9fe35d42b..dfacb60db 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -3,12 +3,14 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import enum from io import BytesIO +import logging import os import os.path as osp from pathlib import Path from stat import S_ISLNK, ST_MODE -import shutil +import subprocess import tempfile import pytest @@ -34,19 +36,97 @@ HOOKS_SHEBANG = "#!/usr/bin/env sh\n" +log = logging.getLogger(__name__) -def _found_in(cmd, directory): - """Check if a command is resolved in a directory (without following symlinks).""" - path = shutil.which(cmd) - return path and Path(path).parent == Path(directory) +class _WinBashMeta(enum.EnumMeta): + """Metaclass allowing :class:`_WinBash` custom behavior when called.""" -is_win_without_bash = os.name == "nt" and not shutil.which("bash.exe") + def __call__(cls): + return cls._check() -is_win_with_wsl_bash = os.name == "nt" and _found_in( - cmd="bash.exe", - directory=Path(os.getenv("WINDIR")) / "System32", -) + +@enum.unique +class _WinBash(enum.Enum, metaclass=_WinBashMeta): + """Status of bash.exe for native Windows. Affects which commit hook tests can pass. + + Call ``_WinBash()`` to check the status. + + This can't be reliably discovered using :func:`shutil.which`, as that approximates + how a shell is expected to search for an executable. On Windows, there are major + differences between how executables are found by a shell and otherwise. (This is the + cmd.exe Windows shell and should not be confused with bash.exe.) Our run_commit_hook + function in GitPython uses subprocess.Popen, including to run bash.exe on Windows. + It does not pass shell=True (and should not). Popen calls CreateProcessW, which + searches several locations prior to using the PATH environment variable. It is + expected to search the System32 directory, even if another directory containing the + executable precedes it in PATH. (There are other differences, less relevant here.) + When WSL is installed, even if no WSL *systems* are installed, bash.exe exists in + System32, and Popen finds it even if another bash.exe precedes it in PATH, as + happens on CI. If WSL is absent, System32 may still have bash.exe, as Windows users + and administrators occasionally copy executables there in lieu of extending PATH. + """ + + INAPPLICABLE = enum.auto() + """This system is not native Windows: either not Windows at all, or Cygwin.""" + + ABSENT = enum.auto() + """No command for ``bash.exe`` is found on the system.""" + + NATIVE = enum.auto() + """Running ``bash.exe`` operates outside any WSL environment (as with Git Bash).""" + + WSL = enum.auto() + """Running ``bash.exe`` runs bash on a WSL system.""" + + WSL_NO_DISTRO = enum.auto() + """Running ``bash.exe` tries to run bash on a WSL system, but none exists.""" + + ERROR_WHILE_CHECKING = enum.auto() + """Could not determine the status. + + This should not trigger a skip or xfail, as it typically indicates either a fixable + problem on the test machine, such as an "Insufficient system resources exist to + complete the requested service" error starting WSL, or a bug in this detection code. + ``ERROR_WHILE_CHECKING.error_or_process`` has details about the most recent failure. + """ + + @classmethod + def _check(cls): + if os.name != "nt": + return cls.INAPPLICABLE + + try: + # Print rather than returning the test command's exit status so that if a + # failure occurs before we even get to this point, we will detect it. + script = 'test -e /proc/sys/fs/binfmt_misc/WSLInterop; echo "$?"' + command = ["bash.exe", "-c", script] + proc = subprocess.run(command, capture_output=True, check=True, text=True) + except FileNotFoundError: + return cls.ABSENT + except OSError as error: + return cls._error(error) + except subprocess.CalledProcessError as error: + no_distro_message = "Windows Subsystem for Linux has no installed distributions." + if error.returncode == 1 and error.stdout.startswith(no_distro_message): + return cls.WSL_NO_DISTRO + return cls._error(error) + + status = proc.stdout.rstrip() + if status == "0": + return cls.WSL + if status == "1": + return cls.NATIVE + return cls._error(proc) + + @classmethod + def _error(cls, error_or_process): + if isinstance(error_or_process, subprocess.CompletedProcess): + log.error("Strange output checking WSL status: %s", error_or_process.stdout) + else: + log.error("Error running bash.exe to check WSL status: %s", error_or_process) + cls.ERROR_WHILE_CHECKING.error_or_process = error_or_process + return cls.ERROR_WHILE_CHECKING def _make_hook(git_dir, name, content, make_exec=True): @@ -917,12 +997,22 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + @pytest.mark.xfail( + _WinBash() is _WinBash.WSL_NO_DISTRO, + reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + raises=HookExecutionError, + ) @with_rw_repo("HEAD", bare=True) def test_pre_commit_hook_success(self, rw_repo): index = rw_repo.index _make_hook(index.repo.git_dir, "pre-commit", "exit 0") index.commit("This should not fail") + @pytest.mark.xfail( + _WinBash() is _WinBash.WSL_NO_DISTRO, + reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + raises=AssertionError, + ) @with_rw_repo("HEAD", bare=True) def test_pre_commit_hook_fail(self, rw_repo): index = rw_repo.index @@ -930,7 +1020,7 @@ def test_pre_commit_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if is_win_without_bash: + if _WinBash() is _WinBash.ABSENT: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") @@ -946,10 +1036,15 @@ def test_pre_commit_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @pytest.mark.xfail( - is_win_without_bash or is_win_with_wsl_bash, + _WinBash() in {_WinBash.ABSENT, _WinBash.WSL}, reason="Specifically seems to fail on WSL bash (in spite of #1399)", raises=AssertionError, ) + @pytest.mark.xfail( + _WinBash() is _WinBash.WSL_NO_DISTRO, + reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + raises=HookExecutionError, + ) @with_rw_repo("HEAD", bare=True) def test_commit_msg_hook_success(self, rw_repo): commit_message = "commit default head by Frèderic Çaufl€" @@ -963,6 +1058,11 @@ def test_commit_msg_hook_success(self, rw_repo): new_commit = index.commit(commit_message) self.assertEqual(new_commit.message, "{} {}".format(commit_message, from_hook_message)) + @pytest.mark.xfail( + _WinBash() is _WinBash.WSL_NO_DISTRO, + reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + raises=AssertionError, + ) @with_rw_repo("HEAD", bare=True) def test_commit_msg_hook_fail(self, rw_repo): index = rw_repo.index @@ -970,7 +1070,7 @@ def test_commit_msg_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if is_win_without_bash: + if _WinBash() is _WinBash.ABSENT: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") From b21535729ca695e47c52086abe390ca4e6792ae3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 23 Nov 2023 22:07:06 -0500 Subject: [PATCH 0872/1790] Simplify/clarify bash.exe check for hook tests; do it only once - Make the design of the enum simpler. - Move some informaton to the check method's docstring. - Call the method once instead of in each decoration. Doing the check only once makes things a little faster, but the more important reason is that the checks can become stale very quickly in some situations. This is unlikely to be an issue on CI, but locally WSL may fail (or not fail) to start up when bash.exe is used in a check, then not fail (or fail) when actaully needed in the test, if the reason it fails is due to resource usage, such as most RAM being used on the system. Although this seems like a reason to do the check multiple times, doing it multiple times in the decorations still does it ahead of when any of the tests is actually run. In contrast, with the change here, we may get outdated results by the time the tests run, but the xfail effects of the check are always consistent with each other and are no longer written in a way that suggests they are more current than they really are. --- test/test_index.py | 73 ++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index dfacb60db..9a520275c 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -39,32 +39,11 @@ log = logging.getLogger(__name__) -class _WinBashMeta(enum.EnumMeta): - """Metaclass allowing :class:`_WinBash` custom behavior when called.""" - - def __call__(cls): - return cls._check() - - @enum.unique -class _WinBash(enum.Enum, metaclass=_WinBashMeta): +class _WinBashStatus(enum.Enum): """Status of bash.exe for native Windows. Affects which commit hook tests can pass. - Call ``_WinBash()`` to check the status. - - This can't be reliably discovered using :func:`shutil.which`, as that approximates - how a shell is expected to search for an executable. On Windows, there are major - differences between how executables are found by a shell and otherwise. (This is the - cmd.exe Windows shell and should not be confused with bash.exe.) Our run_commit_hook - function in GitPython uses subprocess.Popen, including to run bash.exe on Windows. - It does not pass shell=True (and should not). Popen calls CreateProcessW, which - searches several locations prior to using the PATH environment variable. It is - expected to search the System32 directory, even if another directory containing the - executable precedes it in PATH. (There are other differences, less relevant here.) - When WSL is installed, even if no WSL *systems* are installed, bash.exe exists in - System32, and Popen finds it even if another bash.exe precedes it in PATH, as - happens on CI. If WSL is absent, System32 may still have bash.exe, as Windows users - and administrators occasionally copy executables there in lieu of extending PATH. + Call :meth:`check` to check the status. """ INAPPLICABLE = enum.auto() @@ -74,13 +53,13 @@ class _WinBash(enum.Enum, metaclass=_WinBashMeta): """No command for ``bash.exe`` is found on the system.""" NATIVE = enum.auto() - """Running ``bash.exe`` operates outside any WSL environment (as with Git Bash).""" + """Running ``bash.exe`` operates outside any WSL distribution (as with Git Bash).""" WSL = enum.auto() - """Running ``bash.exe`` runs bash on a WSL system.""" + """Running ``bash.exe`` calls ``bash`` in a WSL distribution.""" WSL_NO_DISTRO = enum.auto() - """Running ``bash.exe` tries to run bash on a WSL system, but none exists.""" + """Running ``bash.exe` tries to run bash on a WSL distribution, but none exists.""" ERROR_WHILE_CHECKING = enum.auto() """Could not determine the status. @@ -92,13 +71,34 @@ class _WinBash(enum.Enum, metaclass=_WinBashMeta): """ @classmethod - def _check(cls): + def check(cls): + """Check the status of the ``bash.exe`` :func:`index.fun.run_commit_hook` uses. + + This uses EAFP, attempting to run a command via ``bash.exe``. Which ``bash.exe`` + is used can't be reliably discovered by :func:`shutil.which`, which approximates + how a shell is expected to search for an executable. On Windows, there are major + differences between how executables are found by a shell and otherwise. (This is + the cmd.exe Windows shell, and shouldn't be confused with bash.exe itself. That + the command being looked up also happens to be an interpreter is not relevant.) + + :func:`index.fun.run_commit_hook` uses :class:`subprocess.Popen`, including when + it runs ``bash.exe`` on Windows. It doesn't pass ``shell=True`` (and shouldn't). + On Windows, `Popen` calls ``CreateProcessW``, which searches several locations + prior to using the ``PATH`` environment variable. It is expected to search the + ``System32`` directory, even if another directory containing the executable + precedes it in ``PATH``. (Other differences are less relevant here.) When WSL is + installed, even with no distributions, ``bash.exe`` exists in ``System32``, and + `Popen` finds it even if another ``bash.exe`` precedes it in ``PATH``, as on CI. + If WSL is absent, ``System32`` may still have ``bash.exe``, as Windows users and + administrators occasionally put executables there in lieu of extending ``PATH``. + """ if os.name != "nt": return cls.INAPPLICABLE try: # Print rather than returning the test command's exit status so that if a - # failure occurs before we even get to this point, we will detect it. + # failure occurs before we even get to this point, we will detect it. See + # https://superuser.com/a/1749811 for information on ways to check for WSL. script = 'test -e /proc/sys/fs/binfmt_misc/WSLInterop; echo "$?"' command = ["bash.exe", "-c", script] proc = subprocess.run(command, capture_output=True, check=True, text=True) @@ -129,6 +129,9 @@ def _error(cls, error_or_process): return cls.ERROR_WHILE_CHECKING +_win_bash_status = _WinBashStatus.check() + + def _make_hook(git_dir, name, content, make_exec=True): """A helper to create a hook""" hp = hook_path(name, git_dir) @@ -998,7 +1001,7 @@ class Mocked: self.assertEqual(rel, os.path.relpath(path, root)) @pytest.mark.xfail( - _WinBash() is _WinBash.WSL_NO_DISTRO, + _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=HookExecutionError, ) @@ -1009,7 +1012,7 @@ def test_pre_commit_hook_success(self, rw_repo): index.commit("This should not fail") @pytest.mark.xfail( - _WinBash() is _WinBash.WSL_NO_DISTRO, + _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=AssertionError, ) @@ -1020,7 +1023,7 @@ def test_pre_commit_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if _WinBash() is _WinBash.ABSENT: + if _win_bash_status is _WinBashStatus.ABSENT: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") @@ -1036,12 +1039,12 @@ def test_pre_commit_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @pytest.mark.xfail( - _WinBash() in {_WinBash.ABSENT, _WinBash.WSL}, + _win_bash_status in {_WinBashStatus.ABSENT, _WinBashStatus.WSL}, reason="Specifically seems to fail on WSL bash (in spite of #1399)", raises=AssertionError, ) @pytest.mark.xfail( - _WinBash() is _WinBash.WSL_NO_DISTRO, + _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=HookExecutionError, ) @@ -1059,7 +1062,7 @@ def test_commit_msg_hook_success(self, rw_repo): self.assertEqual(new_commit.message, "{} {}".format(commit_message, from_hook_message)) @pytest.mark.xfail( - _WinBash() is _WinBash.WSL_NO_DISTRO, + _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=AssertionError, ) @@ -1070,7 +1073,7 @@ def test_commit_msg_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if _WinBash() is _WinBash.ABSENT: + if _win_bash_status is _WinBashStatus.ABSENT: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") From cabb5728e75aab3ed73376702e219bd1514ee615 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 23 Nov 2023 22:24:24 -0500 Subject: [PATCH 0873/1790] Temporarily don't install WSL system to test xfail This reverts "Install WSL system on CI for hook tests", commit 9717b8d847b514f761ecaa423226054c73f2a325. Assuming both cases are found to be working, the removed step will either be brought back or replaced with a very similar step that installs a different distribution. --- .github/workflows/pythonpackage.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c2bbc1bbb..d21ea3949 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,12 +35,6 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} - - name: Set up WSL - if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v2.0.2 - with: - distribution: Debian - - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh From 2875ffa083b1e136cd647e6d087a4747aa85a4bc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 23 Nov 2023 22:33:26 -0500 Subject: [PATCH 0874/1790] Put back WSL on Windows CI; pare down debug info This undoes "Temporarily don't install WSL system to test xfail" (cabb572). It keeps Debian as the distribution. (Although the Debian WSL system installs pretty fast already, it may still make sense to try switching to Alpine in the future. But that might need to wait until https://github.com/Vampire/setup-wsl/issues/50 is fixed.) This also removes most of the commands in the WSL debugging step, since the related machinery in test_index.py (_WinBashStatus) seems to be in okay shape, condenses the smaller number of commands that are retained there, and makes much less extensive reduction in the general version and platform information commands as well. This is to make workflow output easier to read, understand, and navigate. --- .github/workflows/cygwin-test.yml | 4 +--- .github/workflows/pythonpackage.yml | 28 ++++++++++------------------ 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 96728e51a..13a01dec4 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -70,9 +70,7 @@ jobs: command -v git python git version python --version - python -c 'import sys; print(sys.platform)' - python -c 'import os; print(os.name)' - python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. + python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' - name: Test with pytest run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d21ea3949..3915296ef 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,6 +35,12 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} + - name: Set up WSL (Windows) + if: startsWith(matrix.os, 'windows') + uses: Vampire/setup-wsl@v2.0.2 + with: + distribution: Debian + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh @@ -62,29 +68,15 @@ jobs: command -v git python git version python --version - python -c 'import sys; print(sys.platform)' - python -c 'import os; print(os.name)' - python -c 'import git; print(git.compat.is_win)' # NOTE: Deprecated. Use os.name directly. + python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' # For debugging hook tests on native Windows systems that may have WSL. - - name: Show where bash.exe may be found + - name: Show bash.exe candidates (Windows) if: startsWith(matrix.os, 'windows') run: | set +e - type -a bash.exe - python -c 'import shutil; print(shutil.which("bash.exe"))' - bash.exe --version - python -c 'import subprocess; p = subprocess.run(["bash.exe", "--version"]); print(f"result: {p!r}")' - bash.exe -c 'echo "$BASH"' - python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", """echo "$BASH" """]); print(f"result: {p!r}")' - bash.exe -c 'echo "$BASH_VERSION"' - python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", """echo "$BASH_VERSION" """]); print(f"result: {p!r}")' - bash.exe -c 'printenv WSL_DISTRO_NAME' - python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", "printenv WSL_DISTRO_NAME"]); print(f"result: {p!r}")' - bash.exe -c 'ls -l /proc/sys/fs/binfmt_misc/WSLInterop' - python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", "ls -l /proc/sys/fs/binfmt_misc/WSLInterop"]); print(f"result: {p!r}")' - bash.exe -c 'uname -a' - python -c 'import subprocess; p = subprocess.run(["bash.exe", "-c", "uname -a"]); print(f"result: {p!r}")' + bash.exe -c 'printenv WSL_DISTRO_NAME; uname -a' + python -c 'import subprocess; subprocess.run(["bash.exe", "-c", "printenv WSL_DISTRO_NAME; uname -a"])' continue-on-error: true - name: Check types with mypy From 0f8cd4ce4a7f942d793a8670257f8738ab18b757 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 23 Nov 2023 23:52:27 -0500 Subject: [PATCH 0875/1790] Treat XPASS status as a test failure This causes full "failure" output to be printed when a test marked xfail unexpectedly passes, and for the test run to be considered failing as a result. The immediate purpose of this change is to facilitate efficient identification of recently introduced wrong or overbroad xfail markings. This behavior may eventually become the pytest default (see #1728 and references therein), and this could be retained even after the current xpassing tests are investigated, to facilitate timely detection of tests marked xfail of code that is newly working. (Individual tests decorated `@pytest.mark.xfail` can still be allowed to unexpectedly pass without it being treated like a test failure, by passing strict=False explicitly.) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7109389d7..5c3117096 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ filterwarnings = "ignore::DeprecationWarning" python_files = "test_*.py" tmp_path_retention_policy = "failed" testpaths = "test" # Space separated list of paths from root e.g test tests doc/testing. +xfail_strict = true # Treat the XPASS status as a test failure (unless strict=False is passed). # --cov coverage # --cov-report term # send report to terminal term-missing -> terminal with line numbers html xml # --cov-report term-missing # to terminal with line numbers From 82c361e7e1a20df96cc9e5c443b2c18fc20b18c9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 24 Nov 2023 02:05:29 -0500 Subject: [PATCH 0876/1790] Correct TestSubmodule.test_rename xfail condition The xfail mark was added in 42a3d74, where the XPASS status on 3.7 was observed but not included in the condition. It turns out this seems to XPASS on a much wider range of versions: all but 3.12. Currently 3.12.0 is the latest stable version and no testing has been done with alpha for 3.13. Most likely whatever causes this test to fail on 3.12 would also apply to 3.13, but because I don't understand the cause, I don't want to guess that, and instead wrote the new condition to expect failure only on 3.12.* versions. --- test/test_submodule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 68dbcc491..1e39924dd 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -950,13 +950,14 @@ def test_remove_norefs(self, rwdir): assert not sm.exists() @pytest.mark.xfail( - os.name == "nt", + os.name == "nt" and (3, 12) <= sys.version_info < (3, 13), reason=( "The sm.move call fails. Submodule.move calls os.renames, which raises:\n" "PermissionError: [WinError 32] " "The process cannot access the file because it is being used by another process: " R"'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\mymodules\myname' " R"-> 'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\renamed\myname'" + "\nThis resembles other Windows errors, but seems only to affect Python 3.12 somehow." ), raises=PermissionError, ) From 0ae5dd19d5e7256b0876987948774b48e207f231 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 24 Nov 2023 02:19:56 -0500 Subject: [PATCH 0877/1790] Revert "Treat XPASS status as a test failure" For its immediate goal, this facilitated identifying the XPASS status of TestSubmodule.test_rename and verifying that the revised xfail condition eliminated it. In the test output, XPASS is written as a failure, with strict xfail behavior noted. Contributors less familiar with marking tests xfail may mistake this to mean some behavior of the code under test was broken. So if strict_xfail is to be enabled, it might be best to do that in a pull request devoted to it, and maybe even to add to the docs, about how to recognize and handle newly xpassing tests. This reverts commit 0f8cd4ce4a7f942d793a8670257f8738ab18b757. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5c3117096..7109389d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,6 @@ filterwarnings = "ignore::DeprecationWarning" python_files = "test_*.py" tmp_path_retention_policy = "failed" testpaths = "test" # Space separated list of paths from root e.g test tests doc/testing. -xfail_strict = true # Treat the XPASS status as a test failure (unless strict=False is passed). # --cov coverage # --cov-report term # send report to terminal term-missing -> terminal with line numbers html xml # --cov-report term-missing # to terminal with line numbers From 0b7ee17849a48ca691eaa3cc5d3eb81a4e6592b7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 24 Nov 2023 20:14:35 -0500 Subject: [PATCH 0878/1790] Refine TestSubmodule.test_rename xfail condition This further improves the condition that was corrected in 82c361e. Testing on Python 3.13.0 alpha 2 shows the same failure as on 3.12 (that I'm at least right now consistently unable to produce on any lower versions). In addition, on both versions of CPython on Windows, the failure seems to consistently resolve if two gc.collect() are placed just above the code that calls sm.move(). A single call is consistently insufficient. I haven't included any such calls in this commit, since the focus here is on fixing xfail markings, and becuse such a change may benefit from being evaluated separately and may merit further accompanying changes. But that this behavior is exhibited on both 3.12 and the 3.13 alpha further supports removing the upper bound on the xfail marking. --- test/test_submodule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 1e39924dd..7fbccf831 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -950,14 +950,14 @@ def test_remove_norefs(self, rwdir): assert not sm.exists() @pytest.mark.xfail( - os.name == "nt" and (3, 12) <= sys.version_info < (3, 13), + os.name == "nt" and sys.version_info >= (3, 12), reason=( "The sm.move call fails. Submodule.move calls os.renames, which raises:\n" "PermissionError: [WinError 32] " "The process cannot access the file because it is being used by another process: " R"'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\mymodules\myname' " R"-> 'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\renamed\myname'" - "\nThis resembles other Windows errors, but seems only to affect Python 3.12 somehow." + "\nThis resembles other Windows errors, but only occurs starting in Python 3.12." ), raises=PermissionError, ) From 8621e892ed5ed05b1a304c9ea25a460f376677bb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 25 Nov 2023 00:01:33 -0500 Subject: [PATCH 0879/1790] Reword comment in _WinBashStatus.check for clarity --- test/test_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 9a520275c..e1bcd5b25 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -96,9 +96,9 @@ def check(cls): return cls.INAPPLICABLE try: - # Print rather than returning the test command's exit status so that if a - # failure occurs before we even get to this point, we will detect it. See - # https://superuser.com/a/1749811 for information on ways to check for WSL. + # Output rather than forwarding the test command's exit status so that if a + # failure occurs before we even get to this point, we will detect it. For + # information on ways to check for WSL, see https://superuser.com/a/1749811. script = 'test -e /proc/sys/fs/binfmt_misc/WSLInterop; echo "$?"' command = ["bash.exe", "-c", script] proc = subprocess.run(command, capture_output=True, check=True, text=True) From 7ff3cee63c53520650db14bdf6e55551b8848567 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 25 Nov 2023 00:23:40 -0500 Subject: [PATCH 0880/1790] Make _WinBashStatus instances carry all their info This changes _WinBashStatus from an enum to a sum type so that, rather than having a global _WinBashStatus.ERROR_WHILE_CHECKING object whose error_or_process attribute may be absent and, if present, carries an object set on the most recent call to check(), we get a _WinBashStatus.ErrorWhileChecking instance that carries an error_or_process attribute specific to the call. Ordinarily this would not make a difference, other than that the global attribute usage was confusing in the code, because typically _WinBashStatus.check() is called only once. However, when debugging, having the old attribute on the same object be clobbered is undesirable. This adds the sumtypes library as a test dependency, to allow the enum to be rewritten as a sum type without loss of clarity. This is a development-only dependency; the main dependencies are unchanged. --- test-requirements.txt | 1 + test/test_index.py | 66 ++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7cfb977a1..fcdc93c1d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,3 +9,4 @@ pytest-cov pytest-instafail pytest-mock pytest-sugar +sumtypes diff --git a/test/test_index.py b/test/test_index.py index e1bcd5b25..f60a86309 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -3,7 +3,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import enum from io import BytesIO import logging import os @@ -14,6 +13,7 @@ import tempfile import pytest +from sumtypes import constructor, sumtype from git import ( IndexFile, @@ -39,35 +39,34 @@ log = logging.getLogger(__name__) -@enum.unique -class _WinBashStatus(enum.Enum): +@sumtype +class _WinBashStatus: """Status of bash.exe for native Windows. Affects which commit hook tests can pass. Call :meth:`check` to check the status. """ - INAPPLICABLE = enum.auto() + Inapplicable = constructor() """This system is not native Windows: either not Windows at all, or Cygwin.""" - ABSENT = enum.auto() + Absent = constructor() """No command for ``bash.exe`` is found on the system.""" - NATIVE = enum.auto() + Native = constructor() """Running ``bash.exe`` operates outside any WSL distribution (as with Git Bash).""" - WSL = enum.auto() + Wsl = constructor() """Running ``bash.exe`` calls ``bash`` in a WSL distribution.""" - WSL_NO_DISTRO = enum.auto() + WslNoDistro = constructor() """Running ``bash.exe` tries to run bash on a WSL distribution, but none exists.""" - ERROR_WHILE_CHECKING = enum.auto() + ErrorWhileChecking = constructor("error_or_process") """Could not determine the status. This should not trigger a skip or xfail, as it typically indicates either a fixable problem on the test machine, such as an "Insufficient system resources exist to complete the requested service" error starting WSL, or a bug in this detection code. - ``ERROR_WHILE_CHECKING.error_or_process`` has details about the most recent failure. """ @classmethod @@ -93,7 +92,13 @@ def check(cls): administrators occasionally put executables there in lieu of extending ``PATH``. """ if os.name != "nt": - return cls.INAPPLICABLE + return cls.Inapplicable() + + no_distro_message = "Windows Subsystem for Linux has no installed distributions." + + def error_running_bash(error): + log.error("Error running bash.exe to check WSL status: %s", error) + return cls.ErrorWhileChecking(error) try: # Output rather than forwarding the test command's exit status so that if a @@ -103,30 +108,21 @@ def check(cls): command = ["bash.exe", "-c", script] proc = subprocess.run(command, capture_output=True, check=True, text=True) except FileNotFoundError: - return cls.ABSENT + return cls.Absent() except OSError as error: - return cls._error(error) + return error_running_bash(error) except subprocess.CalledProcessError as error: - no_distro_message = "Windows Subsystem for Linux has no installed distributions." if error.returncode == 1 and error.stdout.startswith(no_distro_message): - return cls.WSL_NO_DISTRO - return cls._error(error) + return cls.WslNoDistro() + return error_running_bash(error) status = proc.stdout.rstrip() if status == "0": - return cls.WSL + return cls.Wsl() if status == "1": - return cls.NATIVE - return cls._error(proc) - - @classmethod - def _error(cls, error_or_process): - if isinstance(error_or_process, subprocess.CompletedProcess): - log.error("Strange output checking WSL status: %s", error_or_process.stdout) - else: - log.error("Error running bash.exe to check WSL status: %s", error_or_process) - cls.ERROR_WHILE_CHECKING.error_or_process = error_or_process - return cls.ERROR_WHILE_CHECKING + return cls.Native() + log.error("Strange output checking WSL status: %s", proc.stdout) + return cls.ErrorWhileChecking(proc) _win_bash_status = _WinBashStatus.check() @@ -1001,7 +997,7 @@ class Mocked: self.assertEqual(rel, os.path.relpath(path, root)) @pytest.mark.xfail( - _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, + type(_win_bash_status) is _WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=HookExecutionError, ) @@ -1012,7 +1008,7 @@ def test_pre_commit_hook_success(self, rw_repo): index.commit("This should not fail") @pytest.mark.xfail( - _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, + type(_win_bash_status) is _WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=AssertionError, ) @@ -1023,7 +1019,7 @@ def test_pre_commit_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if _win_bash_status is _WinBashStatus.ABSENT: + if type(_win_bash_status) is _WinBashStatus.Absent: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") @@ -1039,12 +1035,12 @@ def test_pre_commit_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @pytest.mark.xfail( - _win_bash_status in {_WinBashStatus.ABSENT, _WinBashStatus.WSL}, + type(_win_bash_status) in {_WinBashStatus.Absent, _WinBashStatus.Wsl}, reason="Specifically seems to fail on WSL bash (in spite of #1399)", raises=AssertionError, ) @pytest.mark.xfail( - _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, + type(_win_bash_status) is _WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=HookExecutionError, ) @@ -1062,7 +1058,7 @@ def test_commit_msg_hook_success(self, rw_repo): self.assertEqual(new_commit.message, "{} {}".format(commit_message, from_hook_message)) @pytest.mark.xfail( - _win_bash_status is _WinBashStatus.WSL_NO_DISTRO, + type(_win_bash_status) is _WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=AssertionError, ) @@ -1073,7 +1069,7 @@ def test_commit_msg_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if _win_bash_status is _WinBashStatus.ABSENT: + if type(_win_bash_status) is _WinBashStatus.Absent: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") From d5ed266f2208367e4300c29fa4cbda9cd2db0fb9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 25 Nov 2023 05:06:23 -0500 Subject: [PATCH 0881/1790] Use bytes in bash.exe check; retest no-distro case This removes text=True from the subprocess.run call, changing str literals to bytes where appropriate and (less importantly) using "%r" instead of "%s" in log messages so it's clear that printing the repr of a bytes object is, at least for now, intentional. The reason for this is that the encoding of error messages produced by running the WSL bash.exe, when it attempts but fails to use a WSL system, varies based on what error occurred. When no systems are installed, the output can be decoded as UTF-8. When an error from "deeper down" is reported, at least for Bash/Service errors, the output is encoded in UTF-16LE, and attempting to decode it as UTF-8 interleaves lots of null characters in the best case. With a bytes object, loss of information is avoided, and it is clear on inspection that the output requires decoding. The most common case of such an error is *probably*: Insufficient system resources exist to complete the requested service. Error code: Bash/Service/CreateInstance/CreateVm/HCS/0x800705aa However, that is tricky to produce intentionally on some systems. To produce a test error, "wsl --shutdown" can be run repeatedly while a _WinBashStatus.check() call is in progress. This produces: The virtual machine or container was forcefully exited. Error code: Bash/Service/0x80370107 Assuming the output always includes the text "Error code:", it may be feasible to reliably detect which cases it is. This could especially improve the log message. But for now that is not done. In adddition to changing from text to binary mode, this commit also temporarily removes the setup-wsl step from the CI workflow again, to verify on CI that the text-to-binary change doesn't break the WslNoDistro case. Manual testing shows the other cases still work. --- .github/workflows/pythonpackage.yml | 6 ------ test/test_index.py | 13 +++++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3915296ef..cef24799e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,12 +35,6 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} - - name: Set up WSL (Windows) - if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v2.0.2 - with: - distribution: Debian - - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh diff --git a/test/test_index.py b/test/test_index.py index f60a86309..8824d6648 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -94,10 +94,11 @@ def check(cls): if os.name != "nt": return cls.Inapplicable() - no_distro_message = "Windows Subsystem for Linux has no installed distributions." + # Use bytes because messages for different WSL errors use different encodings. + no_distro_message = b"Windows Subsystem for Linux has no installed distributions." def error_running_bash(error): - log.error("Error running bash.exe to check WSL status: %s", error) + log.error("Error running bash.exe to check WSL status: %r", error) return cls.ErrorWhileChecking(error) try: @@ -106,7 +107,7 @@ def error_running_bash(error): # information on ways to check for WSL, see https://superuser.com/a/1749811. script = 'test -e /proc/sys/fs/binfmt_misc/WSLInterop; echo "$?"' command = ["bash.exe", "-c", script] - proc = subprocess.run(command, capture_output=True, check=True, text=True) + proc = subprocess.run(command, capture_output=True, check=True) except FileNotFoundError: return cls.Absent() except OSError as error: @@ -117,11 +118,11 @@ def error_running_bash(error): return error_running_bash(error) status = proc.stdout.rstrip() - if status == "0": + if status == b"0": return cls.Wsl() - if status == "1": + if status == b"1": return cls.Native() - log.error("Strange output checking WSL status: %s", proc.stdout) + log.error("Strange output checking WSL status: %r", proc.stdout) return cls.ErrorWhileChecking(proc) From 496acaac5d2cfcbe4edf8ace1bfae452f281ff15 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 26 Nov 2023 00:27:57 -0500 Subject: [PATCH 0882/1790] Handle multiple encodings for WSL error messages This affects the test suite only. It improves _WinBashStatus. When bash.exe is the WSL wrapper, and it reports an error that prevents bash in a WSL system from ever actually being run, the message may be encoded in a narrow or multibyte encoding, or may instead be in Windows's native UTF-16LE encoding. This was partly handled before, by assuming the message indicating the absence of any installed WSL distribuions could be interpreted as UTF-8, but matching against bytes and not actually decoding it or other error messages. That presented a few problems, which are rememedied here: - It turns out that this "Windows Subsystem for Linux has no installed distributions" message actually can be in UTF-16LE too. This happens when it is part of a message that also reports a textual error code like: Bash/Service/CreateInstance/GetDefaultDistro/WSL_E_DEFAULT_DISTRO_NOT_FOUND Therefore, narrow-encoding matching was not sufficient. - Logged messages were hard to understand, because they were printing the reprs of bytes objects, which sometimes were UTF-16LE and thus had many "\x00" interspersed in them. - The cases were not broken down elegantly. The ErrorWhileChecking case could hold a CompletedProcess, a CalledProcessError, or an OSError. This is now replaced with a WinError case for the rare scenario where CreateProcessW fails in a completely unexpected way, and a CheckError case for when the exit status or output of bash.exe indicates an error other than one we want to handle. CheckError has two attributes: `process` for the CompletedProcess (CalledProcessError is no longer used, and CompletedProcess has `returncode` and other attributes that provide the same info), and `message` for the decoded output. --- test/test_index.py | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 8824d6648..bd38ef3bb 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -61,13 +61,11 @@ class _WinBashStatus: WslNoDistro = constructor() """Running ``bash.exe` tries to run bash on a WSL distribution, but none exists.""" - ErrorWhileChecking = constructor("error_or_process") - """Could not determine the status. + CheckError = constructor("process", "message") + """Running ``bash.exe`` fails in an unexpected error or gives unexpected output.""" - This should not trigger a skip or xfail, as it typically indicates either a fixable - problem on the test machine, such as an "Insufficient system resources exist to - complete the requested service" error starting WSL, or a bug in this detection code. - """ + WinError = constructor("exception") + """``bash.exe`` may exist but can't run. ``CreateProcessW`` fails unexpectedly.""" @classmethod def check(cls): @@ -94,12 +92,7 @@ def check(cls): if os.name != "nt": return cls.Inapplicable() - # Use bytes because messages for different WSL errors use different encodings. - no_distro_message = b"Windows Subsystem for Linux has no installed distributions." - - def error_running_bash(error): - log.error("Error running bash.exe to check WSL status: %r", error) - return cls.ErrorWhileChecking(error) + no_distro_message = "Windows Subsystem for Linux has no installed distributions." try: # Output rather than forwarding the test command's exit status so that if a @@ -107,23 +100,26 @@ def error_running_bash(error): # information on ways to check for WSL, see https://superuser.com/a/1749811. script = 'test -e /proc/sys/fs/binfmt_misc/WSLInterop; echo "$?"' command = ["bash.exe", "-c", script] - proc = subprocess.run(command, capture_output=True, check=True) + process = subprocess.run(command, capture_output=True) except FileNotFoundError: return cls.Absent() except OSError as error: - return error_running_bash(error) - except subprocess.CalledProcessError as error: - if error.returncode == 1 and error.stdout.startswith(no_distro_message): - return cls.WslNoDistro() - return error_running_bash(error) - - status = proc.stdout.rstrip() - if status == b"0": + return cls.WinError(error) + + encoding = "utf-16le" if b"\r\0\n\0" in process.stdout else "utf-8" + text = process.stdout.decode(encoding).rstrip() # stdout includes WSL errors. + + if process.returncode == 1 and text.startswith(no_distro_message): + return cls.WslNoDistro() + if process.returncode != 0: + log.error("Error running bash.exe to check WSL status: %s", text) + return cls.CheckError(process, text) + if text == "0": return cls.Wsl() - if status == b"1": + if text == "1": return cls.Native() - log.error("Strange output checking WSL status: %r", proc.stdout) - return cls.ErrorWhileChecking(proc) + log.error("Strange output checking WSL status: %s", text) + return cls.CheckError(process, text) _win_bash_status = _WinBashStatus.check() From d779a7546f4bf69783cf38be4679a0a229578ef9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 27 Nov 2023 04:35:02 -0500 Subject: [PATCH 0883/1790] Don't assume WSL-related bash.exe error is English Instead of searching for an English sentence from the error message, this searches for the specific aka.ms short URL it contains in all languages, which points to a page about downloading distributions for WSL from the Microsoft Store (Windows Store). The URL is controlled and hosted by Microsoft and intended as a permalink. Thus this approach should be more reliable even for English, as the specific wording of the message might be rephrased over time, so this may have a lower risk of false negatives. Because it only makes sense to give a link for obtaining a distribution in an error message when no distribution is installed already, the risk of false positives should also be fairly low. The URL is included in the output telling the user they have no distributions installed even on systems like Windows Server where the Microsoft Store is not itself available, and even in situations where WSL has successfully downloaded and unpacked a distribution but could not run it the first time because it is using WSL 2 but the necessary virtualization is unavailable. Because these are included among the situations where the hook tests should be marked xfail, it is suitable for the purpose at hand. Furthermore, while this only works if we correctly guess if the encoding is UTF-16LE or not, it should be *fairly* robust against incorrect decoding of the message where a single-byte encoding is treated as UTF-8, or UTF-8 is treated as a single-byte encoding, or one single-byte encoding is treated as another (i.e., wrong ANSI code page). But some related encoding subtleties remain to be resolved. Although reliability is likely improved even for English, to faciliate debugging the WslNoDistro case now carries analogous `process` and `message` arguments to the `CheckError` case. On some Windows systems, wsl.exe exists in System32 but bash.exe does not. This is not inherently a problem for run_commit_hook as it currently stands, which is just trying to use whatever bash.exe is available. (It is not clear that WSL should be preferred.) This is currently so, on some systems, where WSL is not really set up; wsl.exe can be used to do so. This is a situation where no WSL distributions are present, but because the bash.exe WSL wrapper is also absent, it is not a problem. However, I had wrongly claimed in the _WinBashStatus.check docstring that bash.exe is in System32 whenever WSL is present. So this also revises the docstring to say only that this is usually so (plus some further revision for clarity). --- test/test_index.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index bd38ef3bb..a7ca2158b 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -8,6 +8,7 @@ import os import os.path as osp from pathlib import Path +import re from stat import S_ISLNK, ST_MODE import subprocess import tempfile @@ -58,7 +59,7 @@ class _WinBashStatus: Wsl = constructor() """Running ``bash.exe`` calls ``bash`` in a WSL distribution.""" - WslNoDistro = constructor() + WslNoDistro = constructor("process", "message") """Running ``bash.exe` tries to run bash on a WSL distribution, but none exists.""" CheckError = constructor("process", "message") @@ -80,20 +81,18 @@ def check(cls): :func:`index.fun.run_commit_hook` uses :class:`subprocess.Popen`, including when it runs ``bash.exe`` on Windows. It doesn't pass ``shell=True`` (and shouldn't). - On Windows, `Popen` calls ``CreateProcessW``, which searches several locations - prior to using the ``PATH`` environment variable. It is expected to search the - ``System32`` directory, even if another directory containing the executable - precedes it in ``PATH``. (Other differences are less relevant here.) When WSL is - installed, even with no distributions, ``bash.exe`` exists in ``System32``, and - `Popen` finds it even if another ``bash.exe`` precedes it in ``PATH``, as on CI. - If WSL is absent, ``System32`` may still have ``bash.exe``, as Windows users and + On Windows, `Popen` calls ``CreateProcessW``, which checks some locations before + using the ``PATH`` environment variable. It is expected to try the ``System32`` + directory, even if another directory containing the executable precedes it in + ``PATH``. (Other differences are less relevant here.) When WSL is present, even + with no distributions, ``bash.exe`` usually exists in ``System32``, and `Popen` + finds it even if another ``bash.exe`` precedes it in ``PATH``, as on CI. If WSL + is absent, ``System32`` may still have ``bash.exe``, as Windows users and administrators occasionally put executables there in lieu of extending ``PATH``. """ if os.name != "nt": return cls.Inapplicable() - no_distro_message = "Windows Subsystem for Linux has no installed distributions." - try: # Output rather than forwarding the test command's exit status so that if a # failure occurs before we even get to this point, we will detect it. For @@ -106,11 +105,12 @@ def check(cls): except OSError as error: return cls.WinError(error) + # FIXME: When not UTF-16LE: try local ANSI code page, then fall back to UTF-8. encoding = "utf-16le" if b"\r\0\n\0" in process.stdout else "utf-8" text = process.stdout.decode(encoding).rstrip() # stdout includes WSL errors. - if process.returncode == 1 and text.startswith(no_distro_message): - return cls.WslNoDistro() + if process.returncode == 1 and re.search(r"\bhttps://aka.ms/wslstore\b", text): + return cls.WslNoDistro(process, text) if process.returncode != 0: log.error("Error running bash.exe to check WSL status: %s", text) return cls.CheckError(process, text) From 9ac243884ebe2d614f03302104064ef22b2aee0f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 27 Nov 2023 17:09:31 -0500 Subject: [PATCH 0884/1790] Handle encodings better; make the sum type "public" Windows does not have a direct analogue of LANG=C or LC_ALL=C. Some programs give them special treatment, but they do not affect the way localized behavior of the Windows API operates. In particular, the bash.exe WSL wrapper, as well as wsl.exe and wslconfig.exe, do not produce their own localized messages (for errors not originating in a running distribution) when they are set. Windows also provides significant localization through localized versions of Windows, so changing language settings in Windows, even system-wide, does not always produce the same effect that many or most Windows users who use a particular language would experience. Various encodings may appear when bash.exe is WSL-related but gives its own error message. Such a message is often in UTF-16LE, which is what Windows uses internally, and preserves any localization. That is the more well behaved scenario and already was detected; this commit moves, but does not change, the code for that. The situation where it is not UTF-16LE was previously handled by treating it as UTF-8. Because the default strict error treatment was used, this would error out in test discovery in some localized setups, preventing all tests in test_index from running, including the majority of them that are not related to hooks. This fixes that by doing better detection that should decode the mesages correctly most of the time, that should in practice decode them well enough to tell (by the aka.ms URL) if the message is complaining about there being no installed distribution all(?) of the time, and that should avoid breaking unrelated tests even if that can't be done. An English non-UTF-16LE message appears on GitHub Actions CI when no distribution is installed. Testing of this situation on other languages was performed in a virtual machine on a development machine. That the message is output in a narrow character set some of the time when bash.exe produces it appears to be a limitation of the bash.exe wrapper. In particular, with a zh-CN version of Windows (and with the language not changed to anything else), a localized message in Simplified Chinese was correctly printed when running wsl.exe, but running bash.exe produced literal "?" characters in place of Chinese characters (it was not a display or font issue, and they were "?" and not Unicode replacement characters). The change here does not overcome that; the literal "?" characters will be included. But "https://aka.ms/wslstore" is still present if it is an error about not having any distributions, so the correct status is still inferred. For more information on code pages in Windows, see: https://serverfault.com/a/836221 The following alternatives to all or part of the approach taken here were considered but, at least for now, not done, because they would not clearly be simpler or more correct: - Abandoning this "run bash.exe and see what it shows us" approach altogether and instead reimplementing the rules CreateProcessW uses, to find if the bash.exe the system finds is the one in System32, and then, if so, checking the metadata in that executable to determine if it's the WSL wrapper. I believe that would be even more complex to do correctly than it seems; the behavior noted in the WinBashStatus docstring and recent commit messages is not the whole story. The documented search order for CreateProcessW seems not to be followed in some circumstances. One is that the Windows Store version of Python seems always to forgo the usual System32 search that precedes seaching directories in PATH. It looks like there may also be some subtleties in which directories 32-bit builds search in. - Using chardet. Although the chardet library is excellent, it is not clear that the code needed to bridge this highly specific use case to it would be simpler than the code currently in use. Some of the work might still need to be done by other means; when I tested it out for this, this did not detect the UTF-16LE messages as such for English. (They are often also valid UTF-8, because interleaving null characters is, while strange, permitted.) - Also calling wsl.exe and/or wslconfig.exe. It's still necessary to call bash.exe, because it may not be the WSL bash, even on a system with WSL fully set up. Furthermore, those tools' output seem to vary in some complex ways, too. Using only one subprocess for the detection seemed the simplest. Even using "wsl --list" would introduce significant additional logic. Sometimes its output is a list of distributions, sometimes it is an error message, and if WSL is not set up it may be a help message. - Using the Windows API to check for WSL systems. https://learn.microsoft.com/en-us/windows/win32/api/wslapi/ does not currently include functions to list registered distributions. - Attempting to get wsl.exe to produce an English message using Windows API techniques like those used in Locale Emulator. This would be complicated, somewhat unintuitive and error prone to do in Python, and I am not sure how well it would work on a system that does not have an English language pack installed. - Checking on disk for WSL distributions in the places they are most often expected to be. This would intertwine WinBashStatus with deep details of how WSL actually operates, and this seems like the sort of thing that is likely to change in the future. However, there may be a more straightforward way to do this (that is at least as correct and that remains transparent to debug). Especially if the WinBashStatus class remains in test_index for long (rather than just being used to aid in debugging existing test falures and possible subsequent design decisions for making commit hooks work more robustly on Windows in GitPython), then this may be worth revisiting. Thus it is *not* with the intention of treating WinBashStatus as a "stable" part of the test suite that it is renamed from _WinBashStatus. This is instead done because: - Like HOOKS_SHEBANG, it makes sense to import it when figuring out how the tests work or debugging them. - When debugging, it is intended that it be imported to call check() and examine the resulting `process` and `message` information, at least in the CheckError case. --- test/test_index.py | 70 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index a7ca2158b..39408e836 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -41,10 +41,13 @@ @sumtype -class _WinBashStatus: +class WinBashStatus: """Status of bash.exe for native Windows. Affects which commit hook tests can pass. Call :meth:`check` to check the status. + + The :class:`CheckError` and :class:`WinError` cases should not typically be used in + ``skip`` or ``xfail`` mark conditions, because they represent unexpected situations. """ Inapplicable = constructor() @@ -84,10 +87,10 @@ def check(cls): On Windows, `Popen` calls ``CreateProcessW``, which checks some locations before using the ``PATH`` environment variable. It is expected to try the ``System32`` directory, even if another directory containing the executable precedes it in - ``PATH``. (Other differences are less relevant here.) When WSL is present, even - with no distributions, ``bash.exe`` usually exists in ``System32``, and `Popen` - finds it even if another ``bash.exe`` precedes it in ``PATH``, as on CI. If WSL - is absent, ``System32`` may still have ``bash.exe``, as Windows users and + ``PATH``. (The other differences are less relevant here.) When WSL is present, + even with no distributions, ``bash.exe`` usually exists in ``System32``, and + `Popen` finds it even if another ``bash.exe`` precedes it in ``PATH``, as on CI. + If WSL is absent, ``System32`` may still have ``bash.exe``, as Windows users and administrators occasionally put executables there in lieu of extending ``PATH``. """ if os.name != "nt": @@ -105,9 +108,7 @@ def check(cls): except OSError as error: return cls.WinError(error) - # FIXME: When not UTF-16LE: try local ANSI code page, then fall back to UTF-8. - encoding = "utf-16le" if b"\r\0\n\0" in process.stdout else "utf-8" - text = process.stdout.decode(encoding).rstrip() # stdout includes WSL errors. + text = cls._decode(process.stdout).rstrip() # stdout includes WSL's own errors. if process.returncode == 1 and re.search(r"\bhttps://aka.ms/wslstore\b", text): return cls.WslNoDistro(process, text) @@ -121,8 +122,45 @@ def check(cls): log.error("Strange output checking WSL status: %s", text) return cls.CheckError(process, text) + @staticmethod + def _decode(stdout): + """Decode ``bash.exe`` output as best we can. (This is used only on Windows.)""" + # When bash.exe is the WSL wrapper but the output is from WSL itself rather than + # code running in a distribution, the output is often in UTF-16LE, which Windows + # uses internally. The UTF-16LE representation of a Windows-style line ending is + # rarely seen otherwise, so use it to detect this situation. + if b"\r\0\n\0" in stdout: + return stdout.decode("utf-16le") + + import winreg + + # At this point, the output is probably either empty or not UTF-16LE. It's often + # UTF-8 from inside a WSL distro or a non-WSL bash shell. But our test command + # only uses the ASCII subset, so it's safe to guess wrong for that command's + # output. Errors from inside a WSL distro or non-WSL bash.exe are arbitrary, but + # unlike WSL's own messages, go to stderr, not stdout. So we can try the system + # active code page first. (Although console programs usually use the OEM code + # page, the ACP seems more accurate here. For example, on en-US Windows set to + # fr-FR, the message, if not UTF-16LE, is windows-1252, same as the ACP, while + # the OEM code page on such a system defaults to 437, which can't decode it.) + hklm_path = R"SYSTEM\CurrentControlSet\Control\Nls\CodePage" + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, hklm_path) as key: + value, _ = winreg.QueryValueEx(key, "ACP") + try: + return stdout.decode(f"cp{value}") + except UnicodeDecodeError: + pass + except LookupError as error: + log.warning("%s", str(error)) # Message already says "Unknown encoding:". + + # Assume UTF-8. If we don't have valid UTF-8, substitute Unicode replacement + # characters. (For example, on zh-CN Windows set to fr-FR, error messages from + # WSL itself, if not UTF-16LE, are in windows-1252, even though the ACP and OEM + # code pages are 936; decoding as code page 936 or as UTF-8 both have errors.) + return stdout.decode("utf-8", errors="replace") + -_win_bash_status = _WinBashStatus.check() +_win_bash_status = WinBashStatus.check() def _make_hook(git_dir, name, content, make_exec=True): @@ -994,7 +1032,7 @@ class Mocked: self.assertEqual(rel, os.path.relpath(path, root)) @pytest.mark.xfail( - type(_win_bash_status) is _WinBashStatus.WslNoDistro, + type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=HookExecutionError, ) @@ -1005,7 +1043,7 @@ def test_pre_commit_hook_success(self, rw_repo): index.commit("This should not fail") @pytest.mark.xfail( - type(_win_bash_status) is _WinBashStatus.WslNoDistro, + type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=AssertionError, ) @@ -1016,7 +1054,7 @@ def test_pre_commit_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if type(_win_bash_status) is _WinBashStatus.Absent: + if type(_win_bash_status) is WinBashStatus.Absent: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") @@ -1032,12 +1070,12 @@ def test_pre_commit_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @pytest.mark.xfail( - type(_win_bash_status) in {_WinBashStatus.Absent, _WinBashStatus.Wsl}, + type(_win_bash_status) in {WinBashStatus.Absent, WinBashStatus.Wsl}, reason="Specifically seems to fail on WSL bash (in spite of #1399)", raises=AssertionError, ) @pytest.mark.xfail( - type(_win_bash_status) is _WinBashStatus.WslNoDistro, + type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=HookExecutionError, ) @@ -1055,7 +1093,7 @@ def test_commit_msg_hook_success(self, rw_repo): self.assertEqual(new_commit.message, "{} {}".format(commit_message, from_hook_message)) @pytest.mark.xfail( - type(_win_bash_status) is _WinBashStatus.WslNoDistro, + type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe for WSL even with no WSL distro installed", raises=AssertionError, ) @@ -1066,7 +1104,7 @@ def test_commit_msg_hook_fail(self, rw_repo): try: index.commit("This should fail") except HookExecutionError as err: - if type(_win_bash_status) is _WinBashStatus.Absent: + if type(_win_bash_status) is WinBashStatus.Absent: self.assertIsInstance(err.status, OSError) self.assertEqual(err.command, [hp]) self.assertEqual(err.stdout, "") From b07e5c7d997b49d980bf3d2d749839726663b274 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 27 Nov 2023 19:58:08 -0500 Subject: [PATCH 0885/1790] Put back WSL on Windows CI So the Windows test jobs can run the three out of four hook tests that are able to pass with WSL bash, and so that we are able to observe the expected failure on the one that still fails. This undoes the removal of the setup-wsl step in d5ed266, now that the test suite's bash.exe status checking logic, used for xfail marks (and also usable in debugging), is internationalized. The CI step was omitted during some of this process, to make sure WinBashStatus would still detect the specific situation on the GitHub Actions runner that occurs when no WSL distro is available. (This commit thus has much of the same relationship to d5ed266 as 2875ffa had to cabb572.) --- .github/workflows/pythonpackage.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cef24799e..3915296ef 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -35,6 +35,12 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} + - name: Set up WSL (Windows) + if: startsWith(matrix.os, 'windows') + uses: Vampire/setup-wsl@v2.0.2 + with: + distribution: Debian + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh From 3303c740bd9aae3ec2fa6b0f1a750bba9ad2b60e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 28 Nov 2023 15:55:29 -0500 Subject: [PATCH 0886/1790] Improve readability of WinBashStatus class - Factor out the code to get the Windows ACP to a helper function, and comment to explain why it doesn't use locale.getencoding. - Remove nonessential material in the WinBashStatus.check docstring and reword the rest for clarity. - Drop reStructuredText notation in the WinBashStatus docstrings, because in this case it seems to be making them harder to read in the code (we are not generating Sphinx documentation for tests.) - Revise the comments on specific steps in WinBashStatus._decode for accuracy and clarity. --- test/test_index.py | 96 +++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 39408e836..bfdbca867 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -40,58 +40,60 @@ log = logging.getLogger(__name__) +def _get_windows_ansi_encoding(): + """Get the encoding specified by the Windows system-wide ANSI active code page.""" + # locale.getencoding may work but is only in Python 3.11+. Use the registry instead. + import winreg + + hklm_path = R"SYSTEM\CurrentControlSet\Control\Nls\CodePage" + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, hklm_path) as key: + value, _ = winreg.QueryValueEx(key, "ACP") + return f"cp{value}" + + @sumtype class WinBashStatus: """Status of bash.exe for native Windows. Affects which commit hook tests can pass. - Call :meth:`check` to check the status. - - The :class:`CheckError` and :class:`WinError` cases should not typically be used in - ``skip`` or ``xfail`` mark conditions, because they represent unexpected situations. + Call check() to check the status. (CheckError and WinError should not typically be + used to trigger skip or xfail, because they represent unexpected situations.) """ Inapplicable = constructor() """This system is not native Windows: either not Windows at all, or Cygwin.""" Absent = constructor() - """No command for ``bash.exe`` is found on the system.""" + """No command for bash.exe is found on the system.""" Native = constructor() - """Running ``bash.exe`` operates outside any WSL distribution (as with Git Bash).""" + """Running bash.exe operates outside any WSL distribution (as with Git Bash).""" Wsl = constructor() - """Running ``bash.exe`` calls ``bash`` in a WSL distribution.""" + """Running bash.exe calls bash in a WSL distribution.""" WslNoDistro = constructor("process", "message") - """Running ``bash.exe` tries to run bash on a WSL distribution, but none exists.""" + """Running bash.exe tries to run bash on a WSL distribution, but none exists.""" CheckError = constructor("process", "message") - """Running ``bash.exe`` fails in an unexpected error or gives unexpected output.""" + """Running bash.exe fails in an unexpected error or gives unexpected output.""" WinError = constructor("exception") - """``bash.exe`` may exist but can't run. ``CreateProcessW`` fails unexpectedly.""" + """bash.exe may exist but can't run. CreateProcessW fails unexpectedly.""" @classmethod def check(cls): - """Check the status of the ``bash.exe`` :func:`index.fun.run_commit_hook` uses. - - This uses EAFP, attempting to run a command via ``bash.exe``. Which ``bash.exe`` - is used can't be reliably discovered by :func:`shutil.which`, which approximates - how a shell is expected to search for an executable. On Windows, there are major - differences between how executables are found by a shell and otherwise. (This is - the cmd.exe Windows shell, and shouldn't be confused with bash.exe itself. That - the command being looked up also happens to be an interpreter is not relevant.) - - :func:`index.fun.run_commit_hook` uses :class:`subprocess.Popen`, including when - it runs ``bash.exe`` on Windows. It doesn't pass ``shell=True`` (and shouldn't). - On Windows, `Popen` calls ``CreateProcessW``, which checks some locations before - using the ``PATH`` environment variable. It is expected to try the ``System32`` - directory, even if another directory containing the executable precedes it in - ``PATH``. (The other differences are less relevant here.) When WSL is present, - even with no distributions, ``bash.exe`` usually exists in ``System32``, and - `Popen` finds it even if another ``bash.exe`` precedes it in ``PATH``, as on CI. - If WSL is absent, ``System32`` may still have ``bash.exe``, as Windows users and - administrators occasionally put executables there in lieu of extending ``PATH``. + """Check the status of the bash.exe that run_commit_hook will try to use. + + This runs a command with bash.exe and checks the result. On Windows, shell and + non-shell executable search differ; shutil.which often finds the wrong bash.exe. + + run_commit_hook uses Popen, including to run bash.exe on Windows. It doesn't + pass shell=True (and shouldn't). On Windows, Popen calls CreateProcessW, which + checks some locations before using the PATH environment variable. It is expected + to try System32, even if another directory with the executable precedes it in + PATH. When WSL is present, even with no distributions, bash.exe usually exists + in System32; Popen finds it even if a shell would run another one, as on CI. + (Without WSL, System32 may still have bash.exe; users sometimes put it there.) """ if os.name != "nt": return cls.Inapplicable() @@ -124,7 +126,7 @@ def check(cls): @staticmethod def _decode(stdout): - """Decode ``bash.exe`` output as best we can. (This is used only on Windows.)""" + """Decode bash.exe output as best we can.""" # When bash.exe is the WSL wrapper but the output is from WSL itself rather than # code running in a distribution, the output is often in UTF-16LE, which Windows # uses internally. The UTF-16LE representation of a Windows-style line ending is @@ -132,31 +134,27 @@ def _decode(stdout): if b"\r\0\n\0" in stdout: return stdout.decode("utf-16le") - import winreg - - # At this point, the output is probably either empty or not UTF-16LE. It's often - # UTF-8 from inside a WSL distro or a non-WSL bash shell. But our test command - # only uses the ASCII subset, so it's safe to guess wrong for that command's - # output. Errors from inside a WSL distro or non-WSL bash.exe are arbitrary, but - # unlike WSL's own messages, go to stderr, not stdout. So we can try the system - # active code page first. (Although console programs usually use the OEM code - # page, the ACP seems more accurate here. For example, on en-US Windows set to - # fr-FR, the message, if not UTF-16LE, is windows-1252, same as the ACP, while - # the OEM code page on such a system defaults to 437, which can't decode it.) - hklm_path = R"SYSTEM\CurrentControlSet\Control\Nls\CodePage" - with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, hklm_path) as key: - value, _ = winreg.QueryValueEx(key, "ACP") + # At this point, the output is either blank or probably not UTF-16LE. It's often + # UTF-8 from inside a WSL distro or non-WSL bash shell. Our test command only + # uses the ASCII subset, so we can safely guess a wrong code page for it. Errors + # from such an environment can contain any text, but unlike WSL's own messages, + # they go to stderr, not stdout. So we can try the system ANSI code page first. + # (Console programs often use the OEM code page, but the ACP seems more accurate + # here. For example, on en-US Windows with the original system code page but the + # display language set to fr-FR, the message, if not UTF-16LE, is windows-1252, + # same as the ACP, while the OEMCP is 437, which can't decode its accents.) + acp = _get_windows_ansi_encoding() try: - return stdout.decode(f"cp{value}") + return stdout.decode(acp) except UnicodeDecodeError: pass except LookupError as error: log.warning("%s", str(error)) # Message already says "Unknown encoding:". - # Assume UTF-8. If we don't have valid UTF-8, substitute Unicode replacement - # characters. (For example, on zh-CN Windows set to fr-FR, error messages from - # WSL itself, if not UTF-16LE, are in windows-1252, even though the ACP and OEM - # code pages are 936; decoding as code page 936 or as UTF-8 both have errors.) + # Assume UTF-8. If invalid, substitute Unicode replacement characters. (For + # example, on zh-CN Windows set to display fr-FR, errors from WSL itself, if not + # UTF-16LE, are in windows-1252, even though the ANSI and OEM code pages both + # default to 936, and decoding as code page 936 or as UTF-8 both have errors.) return stdout.decode("utf-8", errors="replace") From e00fffc918da5cd6c3c749d1d2e59d8ae6835189 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 28 Nov 2023 16:51:10 -0500 Subject: [PATCH 0887/1790] Shorten comments on _decode steps This removes the parenthesized examples from the per-step comments in the WinBashStatus._decode helper. --- test/test_index.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index bfdbca867..cd1c37efc 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -139,10 +139,6 @@ def _decode(stdout): # uses the ASCII subset, so we can safely guess a wrong code page for it. Errors # from such an environment can contain any text, but unlike WSL's own messages, # they go to stderr, not stdout. So we can try the system ANSI code page first. - # (Console programs often use the OEM code page, but the ACP seems more accurate - # here. For example, on en-US Windows with the original system code page but the - # display language set to fr-FR, the message, if not UTF-16LE, is windows-1252, - # same as the ACP, while the OEMCP is 437, which can't decode its accents.) acp = _get_windows_ansi_encoding() try: return stdout.decode(acp) @@ -151,10 +147,7 @@ def _decode(stdout): except LookupError as error: log.warning("%s", str(error)) # Message already says "Unknown encoding:". - # Assume UTF-8. If invalid, substitute Unicode replacement characters. (For - # example, on zh-CN Windows set to display fr-FR, errors from WSL itself, if not - # UTF-16LE, are in windows-1252, even though the ANSI and OEM code pages both - # default to 936, and decoding as code page 936 or as UTF-8 both have errors.) + # Assume UTF-8. If invalid, substitute Unicode replacement characters. return stdout.decode("utf-8", errors="replace") From 6727d0e5d8e6e1ef066781b46122b210d2117497 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 29 Nov 2023 07:09:08 +0800 Subject: [PATCH 0888/1790] Always read metadata files as UTF-8 in setup.py This passes `encoding="utf-8"` to the `open` calls in setup.py, so the readme, version, and requirements files are always read as UTF-8, even on systems whose locale is not UTF-8. This fixes the bug described in #1747 where installation other than from a pre-built wheel would fail on many Windows systems using non-European languages. The specific problem occurred with the README.md file. The requirements files are less likely to contain characters not in the ASCII subset, though maybe they could come to contain them, perhaps in comments. The VERSION file is even less likely to ever contain such characters. Nonetheless, for consistency, because it is a best practice, and because it appears to be the intent of the existing code, encoding="utf=8" is added for opening all of them. This change is tested on a system whose locale uses Windows code page 936. Editable installation, as well as the other affected ways of installing (and building) described in #1747, are now working. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f40f7280c..73d1ae952 100755 --- a/setup.py +++ b/setup.py @@ -7,16 +7,16 @@ import os import sys -with open(os.path.join(os.path.dirname(__file__), "VERSION")) as ver_file: +with open(os.path.join(os.path.dirname(__file__), "VERSION"), encoding="utf-8") as ver_file: VERSION = ver_file.readline().strip() -with open("requirements.txt") as reqs_file: +with open("requirements.txt", encoding="utf-8") as reqs_file: requirements = reqs_file.read().splitlines() -with open("test-requirements.txt") as reqs_file: +with open("test-requirements.txt", encoding="utf-8") as reqs_file: test_requirements = reqs_file.read().splitlines() -with open("README.md") as rm_file: +with open("README.md", encoding="utf-8") as rm_file: long_description = rm_file.read() From e309b35dc216db4596374e1c1a5730e2690deb77 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 13 Nov 2023 22:44:08 -0500 Subject: [PATCH 0889/1790] Mock out lchmod functions in _patch_for_wrapping_test? TestRmtree._patch_for_wrapping_test already mocked out the regular chmod functions in the os module and the Path class, to test what happens when changing permissions cannot fix an error. But there are also, on some systems and Python versions, lchmod versions of these functions. This patches those as well. I am not sure this should really be done. The problem is that calling such functions is fairly likely to raise an exception if it is not properly conditioned on a check for their actual usability, and mocking them out could obscure such a bug. --- test/test_util.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index e5a5a0557..e003fcb05 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -140,9 +140,15 @@ def _patch_for_wrapping_test(self, mocker, hide_windows_known_errors): # git.index.util "replaces" git.util and is what "import git.util" gives us. mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) - # Disable common chmod functions so the callback can't fix a PermissionError. + # Disable some chmod functions so the callback can't fix a PermissionError. mocker.patch.object(os, "chmod") + if hasattr(os, "lchmod"): + # Exists on some operating systems. Mocking out os.chmod doesn't affect it. + mocker.patch.object(os, "lchmod") mocker.patch.object(pathlib.Path, "chmod") + if hasattr(pathlib.Path, "lchmod"): + # Exists on some Python versions. Don't rely on it calling a public chmod. + mocker.patch.object(pathlib.Path, "lchmod") @pytest.mark.skipif( os.name != "nt", From a09e5383cc9bbba290c18c22d925065453b47747 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 28 Nov 2023 20:47:38 -0500 Subject: [PATCH 0890/1790] Don't mock the lchmod functions, and explain why This undoes the mocking of lchmod functions from e309b35, and instead notes why they may be better left alone in the tests. This also rewords the existing comment to better explain the reason for the mocking that is being done. --- test/test_util.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index e003fcb05..68f75aa45 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -140,15 +140,11 @@ def _patch_for_wrapping_test(self, mocker, hide_windows_known_errors): # git.index.util "replaces" git.util and is what "import git.util" gives us. mocker.patch.object(sys.modules["git.util"], "HIDE_WINDOWS_KNOWN_ERRORS", hide_windows_known_errors) - # Disable some chmod functions so the callback can't fix a PermissionError. + # Mock out common chmod functions to simulate PermissionError the callback can't + # fix. (We leave the corresponding lchmod functions alone. If they're used, it's + # more important we detect any failures from inadequate compatibility checks.) mocker.patch.object(os, "chmod") - if hasattr(os, "lchmod"): - # Exists on some operating systems. Mocking out os.chmod doesn't affect it. - mocker.patch.object(os, "lchmod") mocker.patch.object(pathlib.Path, "chmod") - if hasattr(pathlib.Path, "lchmod"): - # Exists on some Python versions. Don't rely on it calling a public chmod. - mocker.patch.object(pathlib.Path, "lchmod") @pytest.mark.skipif( os.name != "nt", From 2fabe71b7ba6feb5e63795c51bf60a1361ed936d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 28 Nov 2023 23:00:29 -0500 Subject: [PATCH 0891/1790] Further document cygpath test parameter collections --- test/test_util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 68f75aa45..f1ce17bca 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -258,14 +258,15 @@ def _xfail_param(*values, **xfail_kwargs): (R"D:/Apps\fOO", "/cygdrive/d/Apps/fOO"), (R"D:\Apps/123", "/cygdrive/d/Apps/123"), ) +"""Path test cases for cygpath and decygpath, other than extended UNC paths.""" _unc_cygpath_pairs = ( (R"\\?\a:\com", "/cygdrive/a/com"), (R"\\?\a:/com", "/cygdrive/a/com"), (R"\\?\UNC\server\D$\Apps", "//server/D$/Apps"), ) +"""Extended UNC path test cases for cygpath.""" -# Mapping of expected failures for the test_cygpath_ok test. _cygpath_ok_xfails = { # From _norm_cygpath_pairs: (R"C:\Users", "/cygdrive/c/Users"): "/proc/cygdrive/c/Users", @@ -279,9 +280,9 @@ def _xfail_param(*values, **xfail_kwargs): (R"\\?\a:\com", "/cygdrive/a/com"): "/proc/cygdrive/a/com", (R"\\?\a:/com", "/cygdrive/a/com"): "/proc/cygdrive/a/com", } +"""Mapping of expected failures for the test_cygpath_ok test.""" -# Parameter sets for the test_cygpath_ok test. _cygpath_ok_params = [ ( _xfail_param(*case, reason=f"Returns: {_cygpath_ok_xfails[case]!r}", raises=AssertionError) @@ -290,6 +291,7 @@ def _xfail_param(*values, **xfail_kwargs): ) for case in _norm_cygpath_pairs + _unc_cygpath_pairs ] +"""Parameter sets for the test_cygpath_ok test.""" @pytest.mark.skipif(sys.platform != "cygwin", reason="Paths specifically for Cygwin.") From e3597180c74d4d1aae444ca42c1e1afdaec3f19a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 29 Nov 2023 00:14:54 -0500 Subject: [PATCH 0892/1790] Revert "Add xfail marks for IndexFile.from_tree failures" This removes the xfail marks from 8 tests that fail due to #1630, to be fixed in the subsequent commits. This reverts commit 6e477e3a06e4b11d43ea118a9f18cae03fa211fd. --- test/test_docs.py | 11 ++--------- test/test_fun.py | 17 +++-------------- test/test_index.py | 40 ---------------------------------------- test/test_refs.py | 11 ----------- 4 files changed, 5 insertions(+), 74 deletions(-) diff --git a/test/test_docs.py b/test/test_docs.py index 2ff1c794a..2f4b2e8d8 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -8,10 +8,11 @@ import pytest -from git.exc import GitCommandError from test.lib import TestBase from test.lib.helper import with_rw_directory +import os.path + class Tutorials(TestBase): def tearDown(self): @@ -206,14 +207,6 @@ def update(self, op_code, cur_count, max_count=None, message=""): assert sm.module_exists() # The submodule's working tree was checked out by update. # ![14-test_init_repo_object] - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_directory def test_references_and_objects(self, rw_dir): # [1-test_references_and_objects] diff --git a/test/test_fun.py b/test/test_fun.py index 8ea5b7e46..566bc9aae 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -3,13 +3,10 @@ from io import BytesIO from stat import S_IFDIR, S_IFREG, S_IFLNK, S_IXUSR -import os +from os import stat import os.path as osp -import pytest - from git import Git -from git.exc import GitCommandError from git.index import IndexFile from git.index.fun import ( aggressive_tree_merge, @@ -37,14 +34,6 @@ def _assert_index_entries(self, entries, trees): assert (entry.path, entry.stage) in index.entries # END assert entry matches fully - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) def test_aggressive_tree_merge(self): # Head tree with additions, removals and modification compared to its predecessor. odb = self.rorepo.odb @@ -302,12 +291,12 @@ def test_linked_worktree_traversal(self, rw_dir): rw_master.git.worktree("add", worktree_path, branch.name) dotgit = osp.join(worktree_path, ".git") - statbuf = os.stat(dotgit) + statbuf = stat(dotgit) self.assertTrue(statbuf.st_mode & S_IFREG) gitdir = find_worktree_git_dir(dotgit) self.assertIsNotNone(gitdir) - statbuf = os.stat(gitdir) + statbuf = stat(gitdir) self.assertTrue(statbuf.st_mode & S_IFDIR) def test_tree_entries_from_data_with_failing_name_decode_py3(self): diff --git a/test/test_index.py b/test/test_index.py index cd1c37efc..5bf34757a 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -284,14 +284,6 @@ def add_bad_blob(): except Exception as ex: assert "index.lock' could not be obtained" not in str(ex) - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_repo("0.1.6") def test_index_file_from_tree(self, rw_repo): common_ancestor_sha = "5117c9c8a4d3af19a9958677e45cda9269de1541" @@ -342,14 +334,6 @@ def test_index_file_from_tree(self, rw_repo): # END for each blob self.assertEqual(num_blobs, len(three_way_index.entries)) - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_repo("0.1.6") def test_index_merge_tree(self, rw_repo): # A bit out of place, but we need a different repo for this: @@ -412,14 +396,6 @@ def test_index_merge_tree(self, rw_repo): self.assertEqual(len(unmerged_blobs), 1) self.assertEqual(list(unmerged_blobs.keys())[0], manifest_key[0]) - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_repo("0.1.6") def test_index_file_diffing(self, rw_repo): # Default Index instance points to our index. @@ -554,14 +530,6 @@ def _count_existing(self, repo, files): # END num existing helper - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): index = rw_repo.index @@ -915,14 +883,6 @@ def make_paths(): for absfile in absfiles: assert osp.isfile(absfile) - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_repo("HEAD") def test_compare_write_tree(self, rw_repo): """Test writing all trees, comparing them for equality.""" diff --git a/test/test_refs.py b/test/test_refs.py index a1573c11b..6ee385007 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -4,11 +4,8 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from itertools import chain -import os from pathlib import Path -import pytest - from git import ( Reference, Head, @@ -218,14 +215,6 @@ def test_head_checkout_detached_head(self, rw_repo): assert isinstance(res, SymbolicReference) assert res.name == "HEAD" - @pytest.mark.xfail( - os.name == "nt", - reason=( - "IndexFile.from_tree is broken on Windows (related to NamedTemporaryFile), see #1630.\n" - "'git read-tree --index-output=...' fails with 'fatal: unable to write new index file'." - ), - raises=GitCommandError, - ) @with_rw_repo("0.1.6") def test_head_reset(self, rw_repo): cur_head = rw_repo.head From b12fd4a8ea6a541efb06c45c83617ec25ea7bd8a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 6 Nov 2023 21:08:24 -0500 Subject: [PATCH 0893/1790] Let Windows subprocess open or rename onto temp file On Windows, a file created by NamedTemporaryFile cannot be reopened while already open, under most circumstances. This applies to attempts to open it both from the original process and from other processes. This differs from the behavior on other operating systems, and it works this way to help ensure the file can be automatically deleted (since having a file open on Windows usually prevents the file from being deleted, unlike on other OSes). However, this can cause problems when NamedTemporaryFile is used to produce a safe filename for accessing with an external process, as IndexFile.from_tree does. On Windows, git subprocess can't open and write to the file. This breaks IndexFile.from_tree on Windows. This is the cause of #1630 -- IndexFile.reset uses IndexFile.from_tree, and the Repo.index property returns an IndexFile instance, so Repo.index.reset fails -- and some other breakages. While passing delete=False to NamedTemporaryFile (and deleting separately) often overcomes that, it won't fix things here, because IndexFile.from_tree uses "git read-tree --index-output=". That needs more than the ability to open the file for write. It needs to be able to create or overwrite the given path by renaming an existing file to it. The description of "--index-output=" in git-read-tree(1) notes: The file must allow to be rename(2)ed into from a temporary file that is created next to the usual index file; typically this means it needs to be on the same filesystem as the index file itself, and you need write permission to the directories the index file and index output file are located in. On Windows, more is required: there must be no currently open file with that path, because on Windows, open files typically cannot be replaced, just as, on Windows, they typically cannot be deleted (nor renamed). This is to say that passing delete=False to NamedTemporaryFile isn't enough to solve this because it merely causes NamedTemporaryFile to behave like the lower-level mkstemp function, leaving the responsibility of deletion to the caller but still *opening* the file -- and while the file is open, the git subprocess can't replace it. Switching to mkstemp, with no further change, would likewise not solve this. This commit is an initial fix for the problem, but not the best fix. On Windows, it closes the temporary file created by NamedTemporaryFile -- so the file can be opened by name, including by a git subprocess, and also overwritten, include by a file move. As currently implemented, this is not ideal, as it creates a race condition, because closing the file actually deletes it. During the time the file does not exist, the filename may end up being reused by another call to NamedTemporaryFile, mkstemp, or similar facility (in any process, written in any language) before the git subprocess can recreate it. Passing delete=False would avoid this, but then deletion would have to be handled separately, and at that point it is not obvious that NamedTemporaryFile is the best facility to use. This fix, though not yet adequate, confirms the impact on #1630. The following 8 tests go from failing to passing due to this change on a local Windows development machine (as listed in the summary at the end of pytest output): - test/test_docs.py:210 Tutorials.test_references_and_objects - test/test_fun.py:37 TestFun.test_aggressive_tree_merge - test/test_index.py:781 TestIndex.test_compare_write_tree - test/test_index.py:294 TestIndex.test_index_file_diffing - test/test_index.py:182 TestIndex.test_index_file_from_tree - test/test_index.py:232 TestIndex.test_index_merge_tree - test/test_index.py:428 TestIndex.test_index_mutation - test/test_refs.py:218 TestRefs.test_head_reset On CI, one test still fails, but later and for an unrelated reason: - test/test_index.py:428 TestIndex.test_index_mutation That `open(fake_symlink_path, "rt")` call raises FileNotFoundError. --- git/index/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/index/base.py b/git/index/base.py index 94437ac88..aa9bfe994 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -362,6 +362,8 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile # works - /tmp/ dirs could be on another device. with ExitStack() as stack: tmp_index = stack.enter_context(tempfile.NamedTemporaryFile(dir=repo.git_dir)) + if os.name == "nt": + tmp_index.close() arg_list.append("--index-output=%s" % tmp_index.name) arg_list.extend(treeish) From 12bbacee90f3810367a2ce986f42c3172c598c58 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 30 Nov 2023 14:38:10 -0500 Subject: [PATCH 0894/1790] Add xfail mark for new test_index_mutation failure The test assumes symlinks are never created on Windows. This often turns out to be correct, because core.symlinks defaults to false on Windows. (On some Windows systems, creating them is a privileged operation; this can be reconfigured, and unprivileged creation of symlinks is often enabled on systems used for development.) However, on a Windows system with core.symlinks set to true, git will create symlinks if it can, when checking out entries committed to a repository as symlinks. GitHub Actions runners for Windows do this ever since https://github.com/actions/runner-images/pull/1186 (the file is now at images/windows/scripts/build/Install-Git.ps1; the `/o:EnableSymlinks=Enabled` option continues to be passed, causing Git for Windows to be installed with core.symlinks set to true in the system scope). For now, this adds an xfail marking to test_index_mutation, for the FileNotFoundError raised when a symlink, which is expected not to be a symlink, is passed to `open`, causing an attempt to open its nonexistent target. (The check itself might bear refinement: as written, it reads the core.symlinks variable from any scope, including the local scope, which at the time of the check will usually be the cloned GitPython directory, where pytest is run.) While adding an import, this commit also improves the grouping and sorting of existing ones. --- test/test_index.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 5bf34757a..2f97f0af8 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -17,17 +17,21 @@ from sumtypes import constructor, sumtype from git import ( + BlobFilter, + Diff, + Git, IndexFile, + Object, Repo, - BlobFilter, - UnmergedEntriesError, Tree, - Object, - Diff, - GitCommandError, +) +from git.exc import ( CheckoutError, + GitCommandError, + HookExecutionError, + InvalidGitRepositoryError, + UnmergedEntriesError, ) -from git.exc import HookExecutionError, InvalidGitRepositoryError from git.index.fun import hook_path from git.index.typ import BaseIndexEntry, IndexEntry from git.objects import Blob @@ -530,6 +534,11 @@ def _count_existing(self, repo, files): # END num existing helper + @pytest.mark.xfail( + os.name == "nt" and Git().config("core.symlinks") == "true", + reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", + raises=FileNotFoundError, + ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): index = rw_repo.index @@ -740,7 +749,7 @@ def mixed_iterator(): # END for each target # END real symlink test - # Add fake symlink and assure it checks-our as symlink. + # Add fake symlink and assure it checks out as a symlink. fake_symlink_relapath = "my_fake_symlink" link_target = "/etc/that" fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) @@ -774,7 +783,7 @@ def mixed_iterator(): os.remove(fake_symlink_path) index.checkout(fake_symlink_path) - # On Windows, we will never get symlinks. + # On Windows, we currently assume we will never get symlinks. if os.name == "nt": # Symlinks should contain the link as text (which is what a # symlink actually is). From 928ca1e9c39ffe3cef9e77cc09efc08dd7d47eee Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 7 Nov 2023 11:25:38 -0500 Subject: [PATCH 0895/1790] Minor formatting consistency improvement --- git/index/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index aa9bfe994..4fa0714d8 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -97,9 +97,8 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): - """ - An Index that can be manipulated using a native implementation in order to save git - command function calls wherever possible. + """An Index that can be manipulated using a native implementation in order to save + git command function calls wherever possible. This provides custom merging facilities allowing to merge without actually changing your index or your working tree. This way you can perform own test-merges based @@ -380,6 +379,7 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile # END index merge handling # UTILITIES + @unbare_repo def _iter_expand_paths(self: "IndexFile", paths: Sequence[PathLike]) -> Iterator[PathLike]: """Expand the directories in list of paths to the corresponding paths accordingly. From 9e5d0aaa0542c0e861dbbb33d48225f4745f56a5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 7 Nov 2023 16:54:44 -0500 Subject: [PATCH 0896/1790] Avoid subprocess-writable temp file race condition This lets the Windows subprocess open or rename onto the temporary file using a more robust approach than in b12fd4a, avoiding the race condition described there, where the filename could be inadvertently reused between deletion and recreation of the file. This creates a context manager helper for the temporary index file used in IndexFile.from_tree, whose implementation differs by operating system: - Delegating straightforwardly to NamedTempoaryFile on POSIX systems where an open file can replaced by having another file renamed to it (just as it can be deleted). - Employing custom logic on Windows, using mkstemp, closing the temporary file without immediately deleting it (so it won't be reused by any process seeking to create a temporary file), and then deleting it on context manager exit. IndexFile.from_tree now calls this helper instead of NamedTemporaryFile. For convenience, the helper provides the path, i.e. the "name", when entered, so tmp_index is now just that path. (At least for now, this helper is implemented as a nonpublic function in the git.index.base module, rather than in the git.index.util module. If it were public in git.index.util like the other facilities there, then some later changes to it, or its later removal, would be a breaking change to GitPython. If it were nonpublic in git.index.util, then this would not be a concern, but it would be unintuitive for it to be accessed from code in the git.index.base module. In the future, one way to address this might be to have one or more nonpublic _util modules with public members. Because it would still be a breaking change to drop existing public util modules, that would be more utility modules in total, so such a change isn't included here just for this one used-once function.) --- git/index/base.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 4fa0714d8..6c6462039 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -6,7 +6,7 @@ """Module containing IndexFile, an Index implementation facilitating all kinds of index manipulations such as querying and merging.""" -from contextlib import ExitStack +import contextlib import datetime import glob from io import BytesIO @@ -67,6 +67,7 @@ BinaryIO, Callable, Dict, + Generator, IO, Iterable, Iterator, @@ -96,6 +97,27 @@ __all__ = ("IndexFile", "CheckoutError", "StageType") +@contextlib.contextmanager +def _named_temporary_file_for_subprocess(directory: PathLike) -> Generator[str, None, None]: + """Create a named temporary file git subprocesses can open, deleting it afterward. + + :param directory: The directory in which the file is created. + + :return: A context manager object that creates the file and provides its name on + entry, and deletes it on exit. + """ + if os.name == "nt": + fd, name = tempfile.mkstemp(dir=directory) + os.close(fd) + try: + yield name + finally: + os.remove(name) + else: + with tempfile.NamedTemporaryFile(dir=directory) as ctx: + yield ctx.name + + class IndexFile(LazyMixin, git_diff.Diffable, Serializable): """An Index that can be manipulated using a native implementation in order to save git command function calls wherever possible. @@ -359,11 +381,9 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile # tmp file created in git home directory to be sure renaming # works - /tmp/ dirs could be on another device. - with ExitStack() as stack: - tmp_index = stack.enter_context(tempfile.NamedTemporaryFile(dir=repo.git_dir)) - if os.name == "nt": - tmp_index.close() - arg_list.append("--index-output=%s" % tmp_index.name) + with contextlib.ExitStack() as stack: + tmp_index = stack.enter_context(_named_temporary_file_for_subprocess(repo.git_dir)) + arg_list.append("--index-output=%s" % tmp_index) arg_list.extend(treeish) # Move current index out of the way - otherwise the merge may fail @@ -373,7 +393,7 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile stack.enter_context(TemporaryFileSwap(join_path_native(repo.git_dir, "index"))) repo.git.read_tree(*arg_list, **kwargs) - index = cls(repo, tmp_index.name) + index = cls(repo, tmp_index) index.entries # Force it to read the file as we will delete the temp-file. return index # END index merge handling From 782c06293f96b4a0b92f4a156a12cc506ea4eaf7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 29 Nov 2023 00:53:37 -0500 Subject: [PATCH 0897/1790] Add myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 3e99ff785..3b97c9473 100644 --- a/AUTHORS +++ b/AUTHORS @@ -52,5 +52,6 @@ Contributors are: -Joseph Hale -Santos Gallegos -Wenhan Zhu +-Eliah Kagan Portions derived from other open source works and are clearly marked. From 2813e94ca80c8e41caaa58ff90c063ca0a26a80c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 30 Nov 2023 22:42:05 -0500 Subject: [PATCH 0898/1790] Add macOS test jobs to CI matrix This now tests on macOS in as well as Ubuntu and Windows. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3915296ef..273e63668 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false From 789baa1c205411545b1105035e773e795ab30302 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 30 Nov 2023 22:53:32 -0500 Subject: [PATCH 0899/1790] Use macOS 13 on CI "macos-latest" is currently macOS 12. Using macOS 13 may be faster. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 273e63668..003ed92c1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-13", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false From f6f335ffcd55af5367692d338d78cf0465c34514 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 30 Nov 2023 23:24:21 -0500 Subject: [PATCH 0900/1790] Temporarily break test_blocking_lock_file on Windows To compare how much extra wait time Windows and macOS usually need. --- test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index f1ce17bca..634bd765d 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -397,7 +397,7 @@ def test_blocking_lock_file(self): self.assertRaises(IOError, wait_lock._obtain_lock) elapsed = time.time() - start extra_time = 0.02 - if os.name == "nt" or sys.platform == "cygwin": + if sys.platform == "cygwin": # FIXME: Put back native Windows check. extra_time *= 6 # NOTE: Indeterministic failures without this... self.assertLess(elapsed, wait_time + extra_time) From c16e4f371b1ee193af92c4ef685315ca1d41e6e9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Dec 2023 00:33:50 -0500 Subject: [PATCH 0901/1790] Restore longer extra time for Windows, and add for macOS This undoes the testing breakage of test_blocking_lock_file for Windows, and gives macOS the same amount of extra time as Windows. I had expected that macOS, even though it needs more time than Ubuntu, might not need as much extra time as Windows. That turned out not to be the case, in the limited testing done so far on CI. Windows, while still slower than Ubuntu and still too slow to reliably pass the test_blocking_lock_file test, was usually faster than macOS and passed more often. These relative timings may turn out not to be a trend and only to apply to the current GHA runners. That's probably okay since the adjustment for macOS wasn't present before and is being added to allow newly introduced macOS CI test jobs to pass. That is all in regard to the very specific issue of the extra time required for the test_blocking_lock_file test after the lock, which makes an assertion about that not taking too long. Regarding the overall time of entire test jobs, macOS 13 seems to have usually been a little faster than macOS 12, so it is retained. Unlike "macos-latest", which currently is macOS 12, "macos-13" will never refer to a later version of the operating system, so the version given in the workflow should be revisited later, at or after the time "macos-latest" becomes a synonym of "macos-13". --- test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index 634bd765d..77332a49d 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -397,7 +397,7 @@ def test_blocking_lock_file(self): self.assertRaises(IOError, wait_lock._obtain_lock) elapsed = time.time() - start extra_time = 0.02 - if sys.platform == "cygwin": # FIXME: Put back native Windows check. + if os.name == "nt" or sys.platform == "cygwin" or sys.platform == "darwin": extra_time *= 6 # NOTE: Indeterministic failures without this... self.assertLess(elapsed, wait_time + extra_time) From d15f891c3127e6b855656b31e04bb1c089f8d54d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Dec 2023 01:07:34 -0500 Subject: [PATCH 0902/1790] macOS needs even more extra time in test_blocking_lock_file As seen in: https://github.com/EliahKagan/GitPython/actions/runs/7056321920/attempts/1 Usually an extra_time of 0.12 (6x) is sufficient for both Windows and macOS, but sometimes macOS needs even more, so this increases it to 0.18 (9x) for macOS. --- test/test_util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 77332a49d..428bd07c3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -397,8 +397,10 @@ def test_blocking_lock_file(self): self.assertRaises(IOError, wait_lock._obtain_lock) elapsed = time.time() - start extra_time = 0.02 - if os.name == "nt" or sys.platform == "cygwin" or sys.platform == "darwin": - extra_time *= 6 # NOTE: Indeterministic failures without this... + if os.name == "nt" or sys.platform == "cygwin": + extra_time *= 6 # Without this, we get indeterministic failures on Windows. + elif sys.platform == "darwin": + extra_time *= 9 # The situation on macOS is similar, but with more delay. self.assertLess(elapsed, wait_time + extra_time) def test_user_id(self): From 78d63d9ecf4cfbeea34dbc464890b97137f3be1b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Dec 2023 02:53:51 -0500 Subject: [PATCH 0903/1790] Let close_fds be True on all platforms Since Python 3.7, subprocess.Popen supports close_fds=True on all platforms, including Windows, and it is the default, including when arguments for standard streams have non-None values passed. 3.7 is the lowest version of Python that GitPython supports. So this omits the close_fds=True argument from the calls where it was present. This has the same effect (in 3.7 and higher) as passing close_fds=True. When the the close_fd argument was added to the Popen call in git.cmd.Git.execute in 1ee2afb, Python 2 was still supported. In Python 2, close_fds defaulted to False. This appears to be the reason it had been passed explicitly. It was conditioned on being on a Unix-like system because having it True on Windows would prevent stdin, stdout, or stderr redirection. --- git/cmd.py | 1 - git/index/fun.py | 1 - 2 files changed, 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index cde6073da..05e199dc0 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1001,7 +1001,6 @@ def execute( stderr=PIPE, stdout=stdout_sink, shell=shell, - close_fds=(os.name == "posix"), # Unsupported on Windows. universal_newlines=universal_newlines, creationflags=PROC_CREATIONFLAGS, **subprocess_kwargs, diff --git a/git/index/fun.py b/git/index/fun.py index 7b3b06269..eaf5f51ff 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -102,7 +102,6 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=index.repo.working_dir, - close_fds=(os.name == "posix"), creationflags=PROC_CREATIONFLAGS, ) except Exception as ex: From 3f21391e9aab5eb89aa3bac6fd32293b129167fc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Dec 2023 14:34:11 -0500 Subject: [PATCH 0904/1790] Remove unused TASKKILL fallback in AutoInterrupt When Git.AutoInterrupt was first implemented in ead94f2, it used os.kill (sending SIGINT to the process to terminate it). This was in 2009, and not all supported versions of Python provided os.kill on Windows, since it was added for Windows in 2.7 and 3.2 (2.6 and 3.1 reached EoL in 2013 and 2012, respectively). Catching AttributeError and running TASKKILL provided a fallback for Python versions in which os.kill was absent on Windows. Since then, all supported versions have os.kill on Windows, and also the code of GitPython, in the try-block, has changed, and no longer uses os.kill. It now contains four attribute access expressions: - proc.terminate. All currently suppported versions of Python (including 3.7, which GitPython still supports, and some before that) have Popen.terminate. - proc.wait. Same situation as proc.terminate. Popen.wait exists. - self._status_code_if_terminate. This is a class attribute of AutoInterrupt, accessible through its instances. - self.status. This is assigned to. AutoInterrupt is slotted, but "status" appears in __slots__, so this isn't an AttributeError either. The "except AttributeError" clause is thus no longer used and can be removed, which is done here. Removing it shortens and simplifies the code, better expresses the logic for how the wrapped process is actually being terminated, and eliminates the need to engage with any subtleties of TASKKILL (and of how it is called) when reading the code to verify its correctness. In addition, because they are now expected always to be available, if somehow an AttributeError managed to be raised in the terminate or wait calls, this would be very strange and we probably shouldn't silently catch that. (Because this AutoInterrupt._terminate method is sometimes called due to __del__ methods running as the interpreter is shutting down, it is possible for some attributes to unexpectedly be set to None, which could cause AttributeError indirectly if another attribute is looked up on the None object; and perhaps on some non-CPython implementations some attributes might even be deleted during shutdown. The _terminate method handles this where relevant. But the TASKKILL fallback removed here seems unrelated to that, which affects module attributes and global variables. The names used in the try-block are proc, status, and self, which are local variables; the except clause itself accessed os.name, which used a global variable and a module attribute, indicating that the intent was not to handle this interpreter shutdown scenario; and this whole issue, while not gone completely, is much less significant since Python 3.4, due to https://docs.python.org/3/whatsnew/3.4.html#whatsnew-pep-442.) --- git/cmd.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 05e199dc0..4dc76556e 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -11,7 +11,7 @@ import logging import os import signal -from subprocess import call, Popen, PIPE, DEVNULL +from subprocess import Popen, PIPE, DEVNULL import subprocess import threading from textwrap import dedent @@ -544,16 +544,6 @@ def _terminate(self) -> None: self.status = self._status_code_if_terminate or status except OSError as ex: log.info("Ignored error after process had died: %r", ex) - except AttributeError: - # Try Windows. - # For some reason, providing None for stdout/stderr still prints something. This is why - # we simply use the shell and redirect to nul. Slower than CreateProcess. The question - # is whether we really want to see all these messages. It's annoying no matter what. - if os.name == "nt": - call( - ("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), - shell=True, - ) # END exception handling def __del__(self) -> None: From 39e2dff35798a08a18a14c03311924def092860c Mon Sep 17 00:00:00 2001 From: Travis Runner Date: Sun, 3 Dec 2023 00:24:28 -0500 Subject: [PATCH 0905/1790] Don't return with operand when conceptually void This changes `return None` to `return` where it appears in functions that are "conceptually void," i.e. cases where a function always returns None, its purpose is never to provide a return value to the caller, and it would be a bug (or at least confusing style) to use the return value or have the call as a subexpression. This is already the prevailing style, so this is a consistency improvement, as well as improving clarity by avoiding giving the impression that "None" is significant in those cases. Of course, functions that sometimes return values other than None do not have "return None" replaced with a bare "return" statement. Those are left alone (when they return None, it means something). For the most part this is straightforward, but: - The handle_stderr local function in IndexFile.checkout is also slightly further refactored to better express what the early return is doing. - The "None" operand is removed as usual in GitConfigParser.read, even though a superclass has a same-named method with a return type that is not None. Usually when this happens, the superclass method returns something like Optional[Something], and it is a case of return type covariance, in which case the None operands should still be written. Here, however, the overriding method does not intend to be an override: it has an incompatible signature and cannot be called as if it were the superclass method, nor is its return type compatible. - The "None" operand is *retained* in git.cmd.handle_process_output even though the return type is annotated as None, because the function also returns the expression `finalizer(process)`. The argument `finalizer` is itself annotated to return None, but it looks like the intent may be to play along with the strange case that `finalizer` returns a non-None value, and forward it. --- git/cmd.py | 4 ++-- git/config.py | 9 +++++---- git/index/base.py | 8 ++++---- git/index/fun.py | 2 +- git/objects/tree.py | 2 +- git/objects/util.py | 4 ++-- git/util.py | 2 +- 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 4dc76556e..5f96e33c9 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -528,13 +528,13 @@ def _terminate(self) -> None: try: if proc.poll() is not None: self.status = self._status_code_if_terminate or proc.poll() - return None + return except OSError as ex: log.info("Ignored error after process had died: %r", ex) # It can be that nothing really exists anymore... if os is None or getattr(os, "kill", None) is None: - return None + return # Try to kill it. try: diff --git a/git/config.py b/git/config.py index 983d71e86..5708a7385 100644 --- a/git/config.py +++ b/git/config.py @@ -203,7 +203,8 @@ def __setitem__(self, key: str, value: _T) -> None: def add(self, key: str, value: Any) -> None: if key not in self: super().__setitem__(key, [value]) - return None + return + super().__getitem__(key).append(value) def setall(self, key: str, values: List[_T]) -> None: @@ -579,7 +580,7 @@ def read(self) -> None: # type: ignore[override] :raise IOError: If a file cannot be handled """ if self._is_initialized: - return None + return self._is_initialized = True files_to_read: List[Union[PathLike, IO]] = [""] @@ -697,7 +698,7 @@ def write(self) -> None: a file lock""" self._assure_writable("write") if not self._dirty: - return None + return if isinstance(self._file_or_files, (list, tuple)): raise AssertionError( @@ -711,7 +712,7 @@ def write(self) -> None: "Skipping write-back of configuration file as include files were merged in." + "Set merge_includes=False to prevent this." ) - return None + return # END stop if we have include files fp = self._file_or_files diff --git a/git/index/base.py b/git/index/base.py index 6c6462039..112ad3feb 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -166,7 +166,7 @@ def _set_cache_(self, attr: str) -> None: except OSError: # In new repositories, there may be no index, which means we are empty. self.entries: Dict[Tuple[PathLike, StageType], IndexEntry] = {} - return None + return # END exception handling try: @@ -1210,9 +1210,9 @@ def checkout( def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLike]) -> None: stderr_IO = proc.stderr if not stderr_IO: - return None # Return early if stderr empty. - else: - stderr_bytes = stderr_IO.read() + return # Return early if stderr empty. + + stderr_bytes = stderr_IO.read() # line contents: stderr = stderr_bytes.decode(defenc) # git-checkout-index: this already exists diff --git a/git/index/fun.py b/git/index/fun.py index eaf5f51ff..97f2c88e6 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -83,7 +83,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): - return None + return env = os.environ.copy() env["GIT_INDEX_FILE"] = safe_decode(str(index.path)) diff --git a/git/objects/tree.py b/git/objects/tree.py index 4d94a5d24..a08adf48b 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -68,7 +68,7 @@ def git_cmp(t1: TreeCacheTup, t2: TreeCacheTup) -> int: def merge_sort(a: List[TreeCacheTup], cmp: Callable[[TreeCacheTup, TreeCacheTup], int]) -> None: if len(a) < 2: - return None + return mid = len(a) // 2 lefthalf = a[:mid] diff --git a/git/objects/util.py b/git/objects/util.py index 3021fec39..3e8f8dcc9 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -500,8 +500,8 @@ def addToStack( depth: int, ) -> None: lst = self._get_intermediate_items(item) - if not lst: # empty list - return None + if not lst: # Empty list + return if branch_first: stack.extendleft(TraverseNT(depth, i, src_item) for i in lst) else: diff --git a/git/util.py b/git/util.py index 01cdcf773..aaf662060 100644 --- a/git/util.py +++ b/git/util.py @@ -644,7 +644,7 @@ def _parse_progress_line(self, line: AnyStr) -> None: self.line_dropped(line_str) # Note: Don't add this line to the other lines, as we have to silently # drop it. - return None + return # END handle op code # Figure out stage. From b9d09c1f5ebceb1810206228e52a6a429cbb463a Mon Sep 17 00:00:00 2001 From: Travis Runner Date: Tue, 5 Dec 2023 00:15:18 -0500 Subject: [PATCH 0906/1790] Group .gitignore entries by purpose - Divide .gitignore entries into groups - Label the groups with comments - Add `__pycache__/` even though it should only contain *.pyc and *.pyo - Add `monkeytype.sqlite3.*` for compressed copies of that database - Remove `/*.egg-info` because `/*egg-info` covers it --- .gitignore | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 191e0e6c3..7765293d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,27 +1,49 @@ +# Cached Python bytecode +__pycache__/ *.py[co] + +# Other caches +.cache/ +.mypy_cache/ +.pytest_cache/ + +# Transient editor files *.swp *~ + +# Editor configuration +nbproject +*.sublime-workspace +/.vscode/ +.idea/ + +# Virtual environments .env/ env/ .venv/ venv/ -/*.egg-info + +# Build output +/*egg-info /lib/GitPython.egg-info -cover/ -.coverage -.coverage.* /build /dist /doc/_build -nbproject -*.sublime-workspace -.DS_Store -/*egg-info + +# Tox builds/environments /.tox -/.vscode/ -.idea/ -.cache/ -.mypy_cache/ -.pytest_cache/ + +# Code coverage output +cover/ +.coverage +.coverage.* + +# Monkeytype output monkeytype.sqlite3 +monkeytype.sqlite3.* + +# Manual command output output.txt + +# Finder metadata +.DS_Store From 1808dddfa64b79a05e678f57bfe87a4b6aca9cfe Mon Sep 17 00:00:00 2001 From: Mario Alvarado Date: Tue, 28 Nov 2023 10:49:28 -0600 Subject: [PATCH 0907/1790] Adding dubious ownership handling --- git/cmd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 5f96e33c9..d67bb1478 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1322,7 +1322,12 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: tokens = header_line.split() if len(tokens) != 3: if not tokens: - raise ValueError("SHA could not be resolved, git returned: %r" % (header_line.strip())) + err_msg = ( + f"SHA is empty, possible dubious ownership in the repository " + f"""at {self._working_dir}.\n If this is unintended run:\n\n """ + f""" "git config --global --add safe.directory {self._working_dir}" """ + ) + raise ValueError(err_msg) else: raise ValueError("SHA %s could not be resolved, git returned: %r" % (tokens[0], header_line.strip())) # END handle actual return value From ad570de497a4512a336eed141726af5b7b7e126b Mon Sep 17 00:00:00 2001 From: Travis Runner Date: Thu, 7 Dec 2023 17:10:17 -0500 Subject: [PATCH 0908/1790] Avoid unsafe assumptions about tempdir content in tests test_new_should_raise_on_invalid_repo_location had previously used tempfile.gettempdir() directly to get a path assumed not to be a valid repository, assuming more than necessary about the directory in which temporary files and directories are created. A newly created temporary directory is now used for that check, instead. test_new_should_raise_on_non_existent_path had assumed no repos/foobar existed relative to the current directory. Typically there would be no such directory, but is unnecessary for the test to rely on this. A newly created temporary directory known to be empty is now used in place of the "repos" component, for that test. --- test/test_repo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_repo.py b/test/test_repo.py index 033deca1e..c68dd074c 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -71,15 +71,19 @@ def tearDown(self): for lfp in glob.glob(_tc_lock_fpaths): if osp.isfile(lfp): raise AssertionError("Previous TC left hanging git-lock file: {}".format(lfp)) + import gc gc.collect() def test_new_should_raise_on_invalid_repo_location(self): - self.assertRaises(InvalidGitRepositoryError, Repo, tempfile.gettempdir()) + with tempfile.TemporaryDirectory() as tdir: + self.assertRaises(InvalidGitRepositoryError, Repo, tdir) def test_new_should_raise_on_non_existent_path(self): - self.assertRaises(NoSuchPathError, Repo, "repos/foobar") + with tempfile.TemporaryDirectory() as tdir: + nonexistent = osp.join(tdir, "foobar") + self.assertRaises(NoSuchPathError, Repo, nonexistent) @with_rw_repo("0.3.2.1") def test_repo_creation_from_different_paths(self, rw_repo): From 9277ff561bcc37c70b9e6e015f623af20216b9e0 Mon Sep 17 00:00:00 2001 From: Travis Runner Date: Thu, 7 Dec 2023 19:06:05 -0500 Subject: [PATCH 0909/1790] Avoid another tempdir content assumption in test test_init was using tempfile.gettempdir() directly to get the location where the hard-coded path repos/foo/bar.git would be used to test repository creation with relative and absolute paths. That reused the same location each time, and also assumed the directory would be usable, which could fail due to previous runs or due to the path being used separately from GitPython's tests. This commit fixes that by using that path inside a temporary directory, known at the start of the test to be empty. Reorganizing the acquision and cleanup logic also has had the effect of causing the test no longer to be skipped due to the logic in git.util.rmtree due to the final cleanup attempt (after all assertions). The directory is now successfully removed on Windows, and the test passes on all platforms. --- test/test_repo.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/test/test_repo.py b/test/test_repo.py index c68dd074c..845fabf15 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -41,7 +41,7 @@ UnsafeProtocolError, ) from git.repo.fun import touch -from git.util import bin_to_hex, cygpath, join_path_native, rmfile, rmtree +from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo @@ -511,13 +511,11 @@ def write(self, b): repo.git.log(n=100, output_stream=TestOutputStream(io.DEFAULT_BUFFER_SIZE)) def test_init(self): - prev_cwd = os.getcwd() - os.chdir(tempfile.gettempdir()) - git_dir_rela = "repos/foo/bar.git" - del_dir_abs = osp.abspath("repos") - git_dir_abs = osp.abspath(git_dir_rela) - try: - # with specific path + with tempfile.TemporaryDirectory() as tdir, cwd(tdir): + git_dir_rela = "repos/foo/bar.git" + git_dir_abs = osp.abspath(git_dir_rela) + + # With specific path for path in (git_dir_rela, git_dir_abs): r = Repo.init(path=path, bare=True) self.assertIsInstance(r, Repo) @@ -527,7 +525,7 @@ def test_init(self): self._assert_empty_repo(r) - # test clone + # Test clone clone_path = path + "_clone" rc = r.clone(clone_path) self._assert_empty_repo(rc) @@ -562,13 +560,6 @@ def test_init(self): assert not r.has_separate_working_tree() self._assert_empty_repo(r) - finally: - try: - rmtree(del_dir_abs) - except OSError: - pass - os.chdir(prev_cwd) - # END restore previous state def test_bare_property(self): self.rorepo.bare From c09ac1ab2c86a87080dcfb2fc0af64cbcc2bc9eb Mon Sep 17 00:00:00 2001 From: Travis Runner Date: Fri, 8 Dec 2023 01:18:54 -0500 Subject: [PATCH 0910/1790] Test InvalidGitRepositoryError in repo subdir A Repo object can of course be constructed from a path to a directory that is the root of an existing repository, and raises InvalidGitRepositoryError on a directory that is outside of any repository. Tests intended to show both conditions already exist. This adds a test to verify that InvalidGitRepositoryError is also raised when an attempt is made to construct a Repo object from a path to a directory that is not the root of a repository but that is known to be located inside an existing git repository. --- test/test_repo.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_repo.py b/test/test_repo.py index 845fabf15..836862b0c 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -77,9 +77,20 @@ def tearDown(self): gc.collect() def test_new_should_raise_on_invalid_repo_location(self): + # Ideally this tests a directory that is outside of any repository. In the rare + # case tempfile.gettempdir() is inside a repo, this still passes, but tests the + # same scenario as test_new_should_raise_on_invalid_repo_location_within_repo. with tempfile.TemporaryDirectory() as tdir: self.assertRaises(InvalidGitRepositoryError, Repo, tdir) + @with_rw_directory + def test_new_should_raise_on_invalid_repo_location_within_repo(self, rw_dir): + repo_dir = osp.join(rw_dir, "repo") + Repo.init(repo_dir) + subdir = osp.join(repo_dir, "subdir") + os.mkdir(subdir) + self.assertRaises(InvalidGitRepositoryError, Repo, subdir) + def test_new_should_raise_on_non_existent_path(self): with tempfile.TemporaryDirectory() as tdir: nonexistent = osp.join(tdir, "foobar") @@ -122,7 +133,7 @@ def test_tree_from_revision(self): self.assertEqual(tree.type, "tree") self.assertEqual(self.rorepo.tree(tree), tree) - # try from invalid revision that does not exist + # Try from an invalid revision that does not exist. self.assertRaises(BadName, self.rorepo.tree, "hello world") def test_pickleable(self): From 86fcb1b54afb77d44298d3f2aebd48d371296f10 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Dec 2023 09:36:13 +0100 Subject: [PATCH 0911/1790] be even more generous with 'extra-time' to wait on MacOS That way, flaky failures are hopefully reliably avoided. --- git/ext/gitdb | 2 +- test/test_util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index ec58b7e55..3d3e9572d 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit ec58b7e55c8fec2479290a22faa293799f6805fc +Subproject commit 3d3e9572dc452fea53d328c101b3d1440bbefe40 diff --git a/test/test_util.py b/test/test_util.py index 428bd07c3..1cb255cfe 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -400,7 +400,7 @@ def test_blocking_lock_file(self): if os.name == "nt" or sys.platform == "cygwin": extra_time *= 6 # Without this, we get indeterministic failures on Windows. elif sys.platform == "darwin": - extra_time *= 9 # The situation on macOS is similar, but with more delay. + extra_time *= 18 # The situation on macOS is similar, but with more delay. self.assertLess(elapsed, wait_time + extra_time) def test_user_id(self): From 24ecf58262eb2e76906689dbc4e28397f4f628dc Mon Sep 17 00:00:00 2001 From: Antoine C Date: Fri, 8 Dec 2023 16:58:24 +0100 Subject: [PATCH 0912/1790] fix #101 --- gitdb/test/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 8fc9e35bf..17906c9c2 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -73,7 +73,7 @@ def test_streams(self): # test deltapackstream dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream + assert dpostream.stream is stream dpostream.read(5) stream._assert() assert stream.bytes == 5 @@ -92,7 +92,7 @@ def test_streams(self): assert istream.size == s istream.size = s * 2 - istream.size == s * 2 + assert istream.size == s * 2 assert istream.type == str_blob_type istream.type = "something" assert istream.type == "something" From e54277fc868a521e360dd9eba5f643321088e813 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Dec 2023 18:59:36 -0500 Subject: [PATCH 0913/1790] Compare types in test_orig_head with "is" This performs an exact type comparison with "is" instead of "==". Most (maybe all?) of the rest of the places where "==" was used have been changed previously, but this on fell through the cracks. Like the None object, type objects use reference-based equality comparison, and the idiomatic ways to check against them are "is" for exactly comparisons and isinstance/issubclass otherwise. --- test/test_refs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_refs.py b/test/test_refs.py index 6ee385007..2656ceab1 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -207,7 +207,7 @@ def test_is_valid(self): assert not SymbolicReference(self.rorepo, "hellothere").is_valid() def test_orig_head(self): - assert type(self.rorepo.head.orig_head()) == SymbolicReference + assert type(self.rorepo.head.orig_head()) is SymbolicReference @with_rw_repo("0.1.6") def test_head_checkout_detached_head(self, rw_repo): From cf5cb8472fa74834274e7077b4c703783ed5112e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Dec 2023 19:04:27 -0500 Subject: [PATCH 0914/1790] Small docstring copyedits - Slightly improve grammar ("it's" -> "its"). - Add a missing space after a "." (between sentences). - Use more consistent hard wrap in a place where 88 was intended. --- git/objects/submodule/base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 9ed1487a5..acf16b556 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -958,7 +958,7 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = raise # END handle undo rename - # Auto-rename submodule if it's name was 'default', that is, the checkout directory. + # Auto-rename submodule if its name was 'default', that is, the checkout directory. if previous_sm_path == self.name: self.rename(module_checkout_path) @@ -976,19 +976,19 @@ def remove( from the .gitmodules file and the entry in the .git/config file. :param module: If True, the checked out module we point to will be deleted as - well.If that module is currently on a commit outside any branch in the + well. If that module is currently on a commit outside any branch in the remote, or if it is ahead of its tracking branch, or if there are modified - or untracked files in its working tree, then the removal will fail. - In case the removal of the repository fails for these reasons, the - submodule status will not have been altered. + or untracked files in its working tree, then the removal will fail. In case + the removal of the repository fails for these reasons, the submodule status + will not have been altered. If this submodule has child modules of its own, these will be deleted prior to touching the direct submodule. :param force: Enforces the deletion of the module even though it contains modifications. This basically enforces a brute-force file system based deletion. :param configuration: If True, the submodule is deleted from the configuration, - otherwise it isn't. Although this should be enabled most of the time, - this flag enables you to safely delete the repository of your submodule. + otherwise it isn't. Although this should be enabled most of the time, this + flag enables you to safely delete the repository of your submodule. :param dry_run: If True, we will not actually do anything, but throw the errors we would usually throw. :return: self From 41d22b277cb0866b3a742e7de63bc33aa881dd04 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Dec 2023 19:08:57 -0500 Subject: [PATCH 0915/1790] Overhaul noqa directives - Remove some noqa directives that are not needed at all anymore. But there may still be some others I did not find. - Fix the code and remove the noqa direcive in places where the change is very simple and does not affect the interface (e.g., what could be taken to be public or not, what symbols are available and what they refer to, etc.), the change makes the code more idiomatic or clearer, and it fixes the flake8 complaint completely. - Turn file-level noqa directives into statement-level ones. A noqa directive at file level, even if it lists one or more specific error/warning numbers, actually suppresses flake8 for the entire file. So the main benefit here is to enable other warnings. But this also makes it clear what is producing the ones we suppress, which makes it easier to know what is intentional and how it may be reasonable to change it in the future. - Move noqa directives at the end of multi-line imports to the top line, where flake8 honors them, and eliminate redundant directives that were added to work around their ineffectiveness. (This is not really separate from the above point, since this is often what allowed file-level directives to be removed.) - Specify the errors/warnings each "noqa" should suppress. That is, this turns every bare "noqa", aside from those eliminated as described above, into a "noqa" for one or more specific errors/warnings. This makes the intent clearer and avoids oversuppression. - Add missing ":" between "noqa" and error/warning numbers. When these were absent, the intention was to suppress only the specifically listed errors/warnings, but the effect was that the listed numbers were ignored and *everything* was suppressed. In a couple of cases it was necessary to change the listed numbers to correct the list to what actually needed to be suppressed. - Write every "noqa" in lower-case (not "NOQA"). There did not appear to be a systematic reason for the different casing, and having them all cased the same makes it easier to avoid mistakes when grepping for them. - Where an import is unused and does not appear intended to be accesed from other modules, and is present only to support code in the same module that is commented out, but whose removal should be considered separately, comment out the statement or part of the statement that imports it as well, rather than writing a "noqa" for the unused import. This applies to only one file, git/objects/util.py. --- git/__init__.py | 27 +++++++++++++-------------- git/cmd.py | 2 +- git/compat.py | 9 ++------- git/index/__init__.py | 6 ++---- git/objects/__init__.py | 20 +++++++++----------- git/objects/util.py | 22 +++++++++------------- git/refs/__init__.py | 13 ++++++------- git/refs/log.py | 4 ++-- git/refs/reference.py | 4 ++-- git/refs/symbolic.py | 5 ++--- git/repo/__init__.py | 4 +--- git/types.py | 14 ++++++-------- git/util.py | 8 ++++---- test/lib/__init__.py | 4 ++-- test/lib/helper.py | 4 ++-- test/test_commit.py | 2 +- test/test_config.py | 2 +- test/test_docs.py | 2 +- test/test_exc.py | 14 +++++++------- test/test_repo.py | 2 +- test/test_submodule.py | 2 +- 21 files changed, 75 insertions(+), 95 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index defc679cb..c6a52ef30 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -3,28 +3,27 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# flake8: noqa # @PydevCodeAnalysisIgnore -from git.exc import * # @NoMove @IgnorePep8 -from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING -from git.types import PathLike - __version__ = "git" +from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING + from gitdb.util import to_hex_sha +from git.exc import * # noqa: F403 # @NoMove @IgnorePep8 +from git.types import PathLike try: from git.compat import safe_decode # @NoMove @IgnorePep8 from git.config import GitConfigParser # @NoMove @IgnorePep8 - from git.objects import * # @NoMove @IgnorePep8 - from git.refs import * # @NoMove @IgnorePep8 - from git.diff import * # @NoMove @IgnorePep8 - from git.db import * # @NoMove @IgnorePep8 + from git.objects import * # noqa: F403 # @NoMove @IgnorePep8 + from git.refs import * # noqa: F403 # @NoMove @IgnorePep8 + from git.diff import * # noqa: F403 # @NoMove @IgnorePep8 + from git.db import * # noqa: F403 # @NoMove @IgnorePep8 from git.cmd import Git # @NoMove @IgnorePep8 from git.repo import Repo # @NoMove @IgnorePep8 - from git.remote import * # @NoMove @IgnorePep8 - from git.index import * # @NoMove @IgnorePep8 + from git.remote import * # noqa: F403 # @NoMove @IgnorePep8 + from git.index import * # noqa: F403 # @NoMove @IgnorePep8 from git.util import ( # @NoMove @IgnorePep8 LockFile, BlockingLockFile, @@ -33,12 +32,12 @@ remove_password_if_present, rmtree, ) -except GitError as _exc: +except GitError as _exc: # noqa: F405 raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc # __all__ must be statically defined by py.typed support # __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] -__all__ = [ +__all__ = [ # noqa: F405 "Actor", "AmbiguousObjectName", "BadName", @@ -127,7 +126,7 @@ def refresh(path: Optional[PathLike] = None) -> None: if not Git.refresh(path=path): return - if not FetchInfo.refresh(): + if not FetchInfo.refresh(): # noqa: F405 return # type: ignore [unreachable] GIT_OK = True diff --git a/git/cmd.py b/git/cmd.py index d67bb1478..8b42dec52 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -965,7 +965,7 @@ def execute( # Only search PATH, not CWD. This must be in the *caller* environment. The "1" can be any value. maybe_patch_caller_env = patch_env("NoDefaultCurrentDirectoryInExePath", "1") else: - cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable + cmd_not_found_exception = FileNotFoundError maybe_patch_caller_env = contextlib.nullcontext() # END handle diff --git a/git/compat.py b/git/compat.py index 64bd38397..6b1b4f547 100644 --- a/git/compat.py +++ b/git/compat.py @@ -5,20 +5,15 @@ """Utilities to help provide compatibility with Python 3.""" -# flake8: noqa - import locale import os import sys -from gitdb.utils.encoding import ( - force_bytes, # @UnusedImport - force_text, # @UnusedImport -) +from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 # @UnusedImport # typing -------------------------------------------------------------------- -from typing import ( +from typing import ( # noqa: F401 Any, AnyStr, Dict, diff --git a/git/index/__init__.py b/git/index/__init__.py index 9954b9e88..c65722cd8 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -3,7 +3,5 @@ """Initialize the index package.""" -# flake8: noqa - -from .base import * -from .typ import * +from .base import * # noqa: F401 F403 +from .typ import * # noqa: F401 F403 diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 9ca430285..fc8d8fbb2 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -3,23 +3,21 @@ """Import all submodules' main classes into the package space.""" -# flake8: noqa - import inspect -from .base import * -from .blob import * -from .commit import * +from .base import * # noqa: F403 +from .blob import * # noqa: F403 +from .commit import * # noqa: F403 from .submodule import util as smutil -from .submodule.base import * -from .submodule.root import * -from .tag import * -from .tree import * +from .submodule.base import * # noqa: F403 +from .submodule.root import * # noqa: F403 +from .tag import * # noqa: F403 +from .tree import * # noqa: F403 # Fix import dependency - add IndexObject to the util module, so that it can be # imported by the submodule.base. -smutil.IndexObject = IndexObject # type: ignore[attr-defined] -smutil.Object = Object # type: ignore[attr-defined] +smutil.IndexObject = IndexObject # type: ignore[attr-defined] # noqa: F405 +smutil.Object = Object # type: ignore[attr-defined] # noqa: F405 del smutil # Must come after submodule was made available. diff --git a/git/objects/util.py b/git/objects/util.py index 3e8f8dcc9..c52dc7564 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -5,20 +5,16 @@ """General utility functions.""" -# flake8: noqa F401 - - from abc import ABC, abstractmethod -import warnings -from git.util import IterableList, IterableObj, Actor - -import re +import calendar from collections import deque - +from datetime import datetime, timedelta, tzinfo from string import digits +import re import time -import calendar -from datetime import datetime, timedelta, tzinfo +import warnings + +from git.util import IterableList, IterableObj, Actor # typing ------------------------------------------------------------ from typing import ( @@ -26,10 +22,10 @@ Callable, Deque, Iterator, - Generic, + # Generic, NamedTuple, overload, - Sequence, # NOQA: F401 + Sequence, TYPE_CHECKING, Tuple, Type, @@ -38,7 +34,7 @@ cast, ) -from git.types import Has_id_attribute, Literal, _T # NOQA: F401 +from git.types import Has_id_attribute, Literal # , _T if TYPE_CHECKING: from io import BytesIO, StringIO diff --git a/git/refs/__init__.py b/git/refs/__init__.py index 3a82b9796..b0233e902 100644 --- a/git/refs/__init__.py +++ b/git/refs/__init__.py @@ -1,13 +1,12 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# flake8: noqa # Import all modules in order, fix the names they require. -from .symbolic import * -from .reference import * -from .head import * -from .tag import * -from .remote import * +from .symbolic import * # noqa: F401 F403 +from .reference import * # noqa: F401 F403 +from .head import * # noqa: F401 F403 +from .tag import * # noqa: F401 F403 +from .remote import * # noqa: F401 F403 -from .log import * +from .log import * # noqa: F401 F403 diff --git a/git/refs/log.py b/git/refs/log.py index aeebac48c..e45798d8a 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -31,9 +31,9 @@ from git.types import PathLike if TYPE_CHECKING: - from git.refs import SymbolicReference from io import BytesIO - from git.config import GitConfigParser, SectionConstraint # NOQA + from git.refs import SymbolicReference + from git.config import GitConfigParser, SectionConstraint # ------------------------------------------------------------------------------ diff --git a/git/refs/reference.py b/git/refs/reference.py index c2ad13bd6..20d42472d 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -10,8 +10,8 @@ # typing ------------------------------------------------------------------ -from typing import Any, Callable, Iterator, Type, Union, TYPE_CHECKING # NOQA -from git.types import Commit_ish, PathLike, _T # NOQA +from typing import Any, Callable, Iterator, Type, Union, TYPE_CHECKING +from git.types import Commit_ish, PathLike, _T if TYPE_CHECKING: from git.repo import Repo diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 84c2057e1..31f959ac2 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -1,7 +1,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.types import PathLike import os from git.compat import defenc @@ -31,8 +30,8 @@ Union, TYPE_CHECKING, cast, -) # NOQA -from git.types import Commit_ish, PathLike # NOQA +) +from git.types import Commit_ish, PathLike if TYPE_CHECKING: from git.repo import Repo diff --git a/git/repo/__init__.py b/git/repo/__init__.py index c01a1e034..a63d77878 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -3,6 +3,4 @@ """Initialize the Repo package.""" -# flake8: noqa - -from .base import Repo as Repo +from .base import Repo as Repo # noqa: F401 diff --git a/git/types.py b/git/types.py index 6f2b7c513..efb393471 100644 --- a/git/types.py +++ b/git/types.py @@ -1,11 +1,9 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# flake8: noqa - import os import sys -from typing import ( +from typing import ( # noqa: F401 Dict, NoReturn, Sequence as Sequence, @@ -16,24 +14,24 @@ Callable, TYPE_CHECKING, TypeVar, -) # noqa: F401 +) if sys.version_info >= (3, 8): - from typing import ( + from typing import ( # noqa: F401 Literal, TypedDict, Protocol, SupportsIndex as SupportsIndex, runtime_checkable, - ) # noqa: F401 + ) else: - from typing_extensions import ( + from typing_extensions import ( # noqa: F401 Literal, SupportsIndex as SupportsIndex, TypedDict, Protocol, runtime_checkable, - ) # noqa: F401 + ) # if sys.version_info >= (3, 10): # from typing import TypeGuard # noqa: F401 diff --git a/git/util.py b/git/util.py index aaf662060..0a5da7d71 100644 --- a/git/util.py +++ b/git/util.py @@ -62,12 +62,9 @@ Has_id_attribute, ) -T_IterableObj = TypeVar("T_IterableObj", bound=Union["IterableObj", "Has_id_attribute"], covariant=True) -# So IterableList[Head] is subtype of IterableList[IterableObj] - # --------------------------------------------------------------------- -from gitdb.util import ( # NOQA @IgnorePep8 +from gitdb.util import ( # noqa: F401 # @IgnorePep8 make_sha, LockedFD, # @UnusedImport file_contents_ro, # @UnusedImport @@ -79,6 +76,9 @@ hex_to_bin, # @UnusedImport ) +T_IterableObj = TypeVar("T_IterableObj", bound=Union["IterableObj", "Has_id_attribute"], covariant=True) +# So IterableList[Head] is subtype of IterableList[IterableObj]. + # NOTE: Some of the unused imports might be used/imported by others. # Handle once test-cases are back up and running. # Most of these are unused here, but are for use by git-python modules so these diff --git a/test/lib/__init__.py b/test/lib/__init__.py index cc1e48483..f96072cb5 100644 --- a/test/lib/__init__.py +++ b/test/lib/__init__.py @@ -3,8 +3,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# flake8: noqa import inspect -from .helper import * + +from .helper import * # noqa: F401 F403 __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] diff --git a/test/lib/helper.py b/test/lib/helper.py index f4e22245b..58d96534a 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -145,7 +145,7 @@ def repo_creator(self): os.chdir(rw_repo.working_dir) try: return func(self, rw_repo) - except: # noqa E722 + except: # noqa: E722 B001 log.info("Keeping repo after failure: %s", repo_dir) repo_dir = None raise @@ -305,7 +305,7 @@ def remote_repo_creator(self): with cwd(rw_repo.working_dir): try: return func(self, rw_repo, rw_daemon_repo) - except: # noqa E722 + except: # noqa: E722 B001 log.info( "Keeping repos after failure: \n rw_repo_dir: %s \n rw_daemon_repo_dir: %s", rw_repo_dir, diff --git a/test/test_commit.py b/test/test_commit.py index b6fb09aef..fddb91c17 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -475,7 +475,7 @@ def test_datetimes(self): commit.authored_datetime, datetime(2009, 10, 8, 18, 17, 5, tzinfo=tzoffset(-7200)), commit.authored_datetime, - ) # noqa + ) self.assertEqual( commit.authored_datetime, datetime(2009, 10, 8, 16, 17, 5, tzinfo=utc), diff --git a/test/test_config.py b/test/test_config.py index c1b26c91f..f493c5672 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -125,7 +125,7 @@ def test_multi_line_config(self): ev = "ruby -e '\n" ev += " system %(git), %(merge-file), %(--marker-size=%L), %(%A), %(%O), %(%B)\n" ev += " b = File.read(%(%A))\n" - ev += " b.sub!(/^<+ .*\\nActiveRecord::Schema\\.define.:version => (\\d+). do\\n=+\\nActiveRecord::Schema\\." # noqa E501 + ev += " b.sub!(/^<+ .*\\nActiveRecord::Schema\\.define.:version => (\\d+). do\\n=+\\nActiveRecord::Schema\\." # noqa: E501 ev += "define.:version => (\\d+). do\\n>+ .*/) do\n" ev += " %(ActiveRecord::Schema.define(:version => #{[$1, $2].max}) do)\n" ev += " end\n" diff --git a/test/test_docs.py b/test/test_docs.py index 2f4b2e8d8..a03a9c71b 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -25,7 +25,7 @@ def tearDown(self): # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " - # "'C:\\Users\\appveyor\\AppData\\Local\\Temp\\1\\test_work_tree_unsupportedryfa60di\\master_repo\\.git\\objects\\pack\\pack-bc9e0787aef9f69e1591ef38ea0a6f566ec66fe3.idx") # noqa E501 + # "'C:\\Users\\appveyor\\AppData\\Local\\Temp\\1\\test_work_tree_unsupportedryfa60di\\master_repo\\.git\\objects\\pack\\pack-bc9e0787aef9f69e1591ef38ea0a6f566ec66fe3.idx") # noqa: E501 @with_rw_directory def test_init_repo_object(self, rw_dir): # [1-test_init_repo_object] diff --git a/test/test_exc.py b/test/test_exc.py index cecd4f342..3f4d0b803 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -40,13 +40,13 @@ ), ) _causes_n_substrings = ( - (None, None), # noqa: E241 @IgnorePep8 - (7, "exit code(7)"), # noqa: E241 @IgnorePep8 - ("Some string", "'Some string'"), # noqa: E241 @IgnorePep8 - ("παλιο string", "'παλιο string'"), # noqa: E241 @IgnorePep8 - (Exception("An exc."), "Exception('An exc.')"), # noqa: E241 @IgnorePep8 - (Exception("Κακια exc."), "Exception('Κακια exc.')"), # noqa: E241 @IgnorePep8 - (object(), " Date: Sun, 10 Dec 2023 02:37:11 -0500 Subject: [PATCH 0916/1790] Document more Git.execute kill_after_timeout limitations See discussion in #1756. --- git/cmd.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 8b42dec52..af748e529 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -879,11 +879,19 @@ def execute( Specifies a timeout in seconds for the git command, after which the process should be killed. This will have no effect if `as_process` is set to True. It is set to None by default and will let the process run until the timeout - is explicitly specified. This feature is not supported on Windows. It's also - worth noting that `kill_after_timeout` uses SIGKILL, which can have negative - side effects on a repository. For example, stale locks in case of ``git gc`` - could render the repository incapable of accepting changes until the lock is - manually removed. + is explicitly specified. Uses of this feature should be carefully + considered, due to the following limitations: + + 1. This feature is not supported at all on Windows. + 2. Effectiveness may vary by operating system. ``ps --ppid`` is used to + enumerate child processes, which is available on most GNU/Linux systems + but not most others. + 3. Deeper descendants do not receive signals, though they may sometimes + terminate as a consequence of their parent processes being killed. + 4. `kill_after_timeout` uses ``SIGKILL``, which can have negative side + effects on a repository. For example, stale locks in case of ``git gc`` + could render the repository incapable of accepting changes until the lock + is manually removed. :param with_stdout: If True, default True, we open stdout on the created process. From 2f017ac879eece80e04fd2a30238f261a6b49fb4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Dec 2023 02:38:05 -0500 Subject: [PATCH 0917/1790] Avoid making it look like kill_process works on Windows This changes the code in Git.execute's local kill_process function, which it uses as the timed callback for kill_after_timeout, to remove code that is unnecessary because kill_process doesn't support Windows, and to avoid giving the false impression that its code could be used unmodified on Windows without serious problems. - Raise AssertionError explicitly if it is called on Windows. This is done with "raise" rather than "assert" so its behavior doesn't vary depending on "-O". - Don't pass process creation flags, because they were 0 except on Windows. - Don't fall back to SIGTERM if Python's signal module doesn't know about SIGKILL. This was specifically for Windows which has no SIGKILL. See #1756 for discussion. --- git/cmd.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index af748e529..27148d3d6 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1015,11 +1015,9 @@ def execute( def kill_process(pid: int) -> None: """Callback to kill a process.""" - p = Popen( - ["ps", "--ppid", str(pid)], - stdout=PIPE, - creationflags=PROC_CREATIONFLAGS, - ) + if os.name == "nt": + raise AssertionError("Bug: This callback would be ineffective and unsafe on Windows, stopping.") + p = Popen(["ps", "--ppid", str(pid)], stdout=PIPE) child_pids = [] if p.stdout is not None: for line in p.stdout: @@ -1028,18 +1026,16 @@ def kill_process(pid: int) -> None: if local_pid.isdigit(): child_pids.append(int(local_pid)) try: - # Windows does not have SIGKILL, so use SIGTERM instead. - sig = getattr(signal, "SIGKILL", signal.SIGTERM) - os.kill(pid, sig) + os.kill(pid, signal.SIGKILL) for child_pid in child_pids: try: - os.kill(child_pid, sig) + os.kill(child_pid, signal.SIGKILL) except OSError: pass kill_check.set() # Tell the main routine that the process was killed. except OSError: - # It is possible that the process gets completed in the duration after timeout - # happens and before we try to kill the process. + # It is possible that the process gets completed in the duration after + # timeout happens and before we try to kill the process. pass return From 36dd418b05f962569fd894d4edf533fc8d7c5bed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 02:10:45 +0000 Subject: [PATCH 0918/1790] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e816355f3..fd73d6dd6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 86402e67e7d999b2b2665dc1029c5e1ccd3ada35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:03:07 +0000 Subject: [PATCH 0919/1790] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73b3902c3..ec7550de2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -21,7 +21,7 @@ jobs: with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From d1cb8c72e4671fd2b027f9326cf32943ba861d67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:03:20 +0000 Subject: [PATCH 0920/1790] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 91dd919e0..f9c5b70b3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,7 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 003ed92c1..e9a7486be 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -30,7 +30,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From d70ba6941cf0f005f3f247c70233f51a0cc35969 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 06:52:35 -0500 Subject: [PATCH 0921/1790] Extract all "import gc" to module level The gc module was already imported at module scope in git/repo/base.py, since f1a82e4 (#555). Importing the top-level git module or any submodule of it runs that import statement. Because the gc module is already imported, reimporting it is fast. Imports that there is no specific reason to do locally should be at module scope. Having them local decreased readability, in part because of how black inserts a black line between them and gc.collect() calls they are imported to allow. An alternative to this change would be to remove the preexisting top-level "import gc" (there is also another one in the test suite) and replace it with a local import as well. I am unsure if that would affect performance and, if so, whether the effect would be good or bad, since the small delay of the import might potentially be less desirable to an applicaion if it occurs while the work of the application is already in progress. If a gc.collect() call runs as a consequence of a finally block or __del__ method being called during interpreter shutdown, then in (very) rare cases the variable may have been set to None. But this does not appear to have been the intent behind making the imports local. More importantly, a local import should not be expected to succeed, or the imported module usable, in such a situation. --- git/objects/submodule/base.py | 3 +-- test/performance/test_commit.py | 3 +-- test/performance/test_streams.py | 3 +-- test/test_base.py | 3 +-- test/test_docs.py | 3 +-- test/test_git.py | 3 +-- test/test_quick_doc.py | 4 ++-- test/test_remote.py | 3 +-- test/test_repo.py | 3 +-- test/test_submodule.py | 3 +-- 10 files changed, 11 insertions(+), 20 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index acf16b556..651d9535c 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,6 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc from io import BytesIO import logging import os @@ -1079,8 +1080,6 @@ def remove( self._clear_cache() wtd = mod.working_tree_dir del mod # Release file-handles (Windows). - import gc - gc.collect() rmtree(str(wtd)) # END delete tree if possible diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index 9e136a6c1..00d768f0a 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -5,6 +5,7 @@ """Performance tests for commits (iteration, traversal, and serialization).""" +import gc from io import BytesIO from time import time import sys @@ -17,8 +18,6 @@ class TestPerformance(TestBigRepoRW, TestCommitSerialization): def tearDown(self): - import gc - gc.collect() # ref with about 100 commits in its history. diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index 9ee7cf5e2..ba5cbe415 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -3,6 +3,7 @@ """Performance tests for data streaming.""" +import gc import os import subprocess import sys @@ -92,8 +93,6 @@ def test_large_data_streaming(self, rwrepo): # del db file so git has something to do. ostream = None - import gc - gc.collect() os.remove(db_file) diff --git a/test/test_base.py b/test/test_base.py index a667b194c..cdf82b74d 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc import os import sys import tempfile @@ -20,8 +21,6 @@ class TestBase(_TestBase): def tearDown(self): - import gc - gc.collect() type_tuples = ( diff --git a/test/test_docs.py b/test/test_docs.py index a03a9c71b..48922eba8 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc import os import sys @@ -16,8 +17,6 @@ class Tutorials(TestBase): def tearDown(self): - import gc - gc.collect() # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last call to it via diff --git a/test/test_git.py b/test/test_git.py index 5986a07da..6d0ae82d0 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc import inspect import logging import os @@ -34,8 +35,6 @@ def setUpClass(cls): cls.git = Git(cls.rorepo.working_dir) def tearDown(self): - import gc - gc.collect() def _assert_logged_for_popen(self, log_watcher, name, value): diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 504dca237..4ef75f4aa 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -1,14 +1,14 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc + from test.lib import TestBase from test.lib.helper import with_rw_directory class QuickDoc(TestBase): def tearDown(self): - import gc - gc.collect() @with_rw_directory diff --git a/test/test_remote.py b/test/test_remote.py index 080718a1c..df6034326 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc import os import os.path as osp from pathlib import Path @@ -105,8 +106,6 @@ def assert_received_message(self): class TestRemote(TestBase): def tearDown(self): - import gc - gc.collect() def _print_fetchhead(self, repo): diff --git a/test/test_repo.py b/test/test_repo.py index 7560c519f..007b8ecb0 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc import glob import io from io import BytesIO @@ -72,8 +73,6 @@ def tearDown(self): if osp.isfile(lfp): raise AssertionError("Previous TC left hanging git-lock file: {}".format(lfp)) - import gc - gc.collect() def test_new_should_raise_on_invalid_repo_location(self): diff --git a/test/test_submodule.py b/test/test_submodule.py index b68ca3fed..852a5ef6f 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -2,6 +2,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib +import gc import os import os.path as osp from pathlib import Path @@ -61,8 +62,6 @@ def update(self, op, cur_count, max_count, message=""): class TestSubmodule(TestBase): def tearDown(self): - import gc - gc.collect() k_subm_current = "c15a6e1923a14bc760851913858a3942a4193cdb" From 43f3b52f17aac918a76a79665a61461e74f4ca1a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 07:26:04 -0500 Subject: [PATCH 0922/1790] Don't install black on Cygwin As of black 23.12.0, installing black builds a wheel on Cygwin, downloading dependencies for the "d" extra (i.e., black[d]), which provides dependencies for running the blackd daemon. This includes dependencies, such as asyncio, whose compilation fails on CI as it is currently set up, fails under some common Cygwin setups (since it needs a C compiler), and even if successful would cause the Cygwin test workflow to take significantly longer to run. --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fcdc93c1d..63fa0a9cd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -black +black ; sys_platform != "cygwin" coverage[toml] ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" From f62df523e0121ddead413da6d6eaf944a39c74e5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 09:15:19 -0500 Subject: [PATCH 0923/1790] Remove TestSubmodule.test_rename xfail mark This is to clearly establish the failure still occurs (since we do not have strict=True set). It is in preparation for working around the problems with a call to gc.collect(). See 82c361e and 0b7ee17 for context. --- test/test_submodule.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 852a5ef6f..e2380d522 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -948,18 +948,6 @@ def test_remove_norefs(self, rwdir): sm.remove() assert not sm.exists() - @pytest.mark.xfail( - os.name == "nt" and sys.version_info >= (3, 12), - reason=( - "The sm.move call fails. Submodule.move calls os.renames, which raises:\n" - "PermissionError: [WinError 32] " - "The process cannot access the file because it is being used by another process: " - R"'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\mymodules\myname' " - R"-> 'C:\Users\ek\AppData\Local\Temp\test_renamekkbznwjp\parent\renamed\myname'" - "\nThis resembles other Windows errors, but only occurs starting in Python 3.12." - ), - raises=PermissionError, - ) @with_rw_directory def test_rename(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) From b66be7ca948307460933939bdbebc8f4ed665299 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 08:04:01 -0500 Subject: [PATCH 0924/1790] Replace xfail with gc.collect in TestSubmodule.test_rename Like the xfail was, this is conditional, being done only in the specific situation the PermissionError occurs. Besides that it does not always run even on Windows (only in 3.12 and later), this resembles various other conditional and non-conditional gc.collect calls. It had previously appeared to me that two calls to gc.collect were required, but I am unable to reproduce that. It may have been specific to how I was running it on my system at that time. The need for only one call may have been brought about by changes to the code in the mean time, but I have tested that only one call appears required even without the changes in #1765. --- test/test_submodule.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index e2380d522..4dc89f98f 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -958,6 +958,12 @@ def test_rename(self, rwdir): assert sm.rename(sm_name) is sm and sm.name == sm_name assert not sm.repo.is_dirty(index=True, working_tree=False, untracked_files=False) + # This is needed to work around a PermissionError on Windows, resembling others, + # except new in Python 3.12. (*Maybe* this could be due to changes in CPython's + # garbage collector detailed in https://github.com/python/cpython/issues/97922.) + if os.name == "nt" and sys.version_info >= (3, 12): + gc.collect() + new_path = "renamed/myname" assert sm.move(new_path).name == new_path From 96cae0002efe5c108be4a3dad0ae2c42069f1ca6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 10:09:34 -0500 Subject: [PATCH 0925/1790] Extract remaining local "import gc" to module level In #1765, I missed one, somehow. This fixes that. --- test/test_diff.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_diff.py b/test/test_diff.py index 1d138a086..87f92f5d1 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import gc import os import os.path as osp import shutil @@ -27,8 +28,6 @@ def setUp(self): self.submodule_dir = tempfile.mkdtemp() def tearDown(self): - import gc - gc.collect() shutil.rmtree(self.repo_dir) shutil.rmtree(self.submodule_dir) From 41fac851fb62b4224e0400be46a3884708d185c7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 19:02:14 -0500 Subject: [PATCH 0926/1790] Avoid mktemp in tests, in straightforward cases The tempfile.mktemp function is deprecated, because of a race condition where the file may be concurrently created between when its name is generated and when it is opened. Other faciliies in the tempfile module overcome this by generating a name, attempting to create the file or directory in a way that guarantees failure if it already existed, and, in the occasional case that it did already exist, generating another name and trying again (stopping after a predefined limit). For further information on mktemp deprecation: - https://docs.python.org/3/library/tempfile.html#tempfile.mktemp - https://github.com/gitpython-developers/smmap/issues/41 The security risk of calls to mktemp in this project's test suite is low. However, it is still best to avoid using it, because it is deprecated, because it is (at least slightly) brittle, and because any use of mktemp looks like a potential security risk and thereby imposes a burden on working with the code (which could potentially be addressed with detailed comments analyzing why it is believed safe in particular cases, but this would typically be more verbose, and at least as challenging to add, as replacing mktemp with a better alternative). This commit replaces *some* uses of mktemp in the test suite: those where it is readily clear how to do so in a way that preserves the code's intent: - Where a name for a temporary directory is generated with mktemp and os.mkdir is called immediately, mkdtemp is now used. - Where a name for a temporary file that is not customized (such as with a prefix) is generated with mktemp, such that the code under test never uses the filename but only the already-open file-like object, TemporaryFile is now used. As the name isn't customized, the test code in these cases does not express an intent to allow the developer to inspect the file after a test failure, so even if the file wasn't guaranteed to be deleted with a finally block or context manager, it is fine to do so. TemporaryFile supports this use case well on all systems including Windows, and automatically deletes the file. - Where a name for a temporary file that *is* customized (such as with a prefix) to reflect the way the test uses it is generated with mktemp, and the test code does not attempt deterministic deletion of the file when an exception would make the test fail, NamedTemporaryFile with delete=False is now used. The original code to remove the file when the test succeeds is modified accordingly to do the same job, and also commented to explain that it is being handled this way to allow the file to be kept and examined when a test failure occurs. Other cases in the test suite should also be feasible to replace, but are left alone for now. Some of them are ambiguous in their intent, with respect to whether the file should be retained after a test failure. Others appear deliberately to avoid creating a file or directory where the code under test should do so, possibly to check that this is done properly. (One way to preserve that latter behavior, while avoiding the weakness of using mktemp and also avoiding inadverently reproducing that weakness by other means, could be to use a path in a temporary directory made for the test.) This commit also doesn't address the one use of mktemp in the code under test (i.e., outside the test suite, inside the git module). --- test/lib/helper.py | 3 +- test/performance/lib.py | 3 +- test/test_base.py | 5 ++-- test/test_reflog.py | 7 ++--- test/test_repo.py | 5 ++-- test/test_util.py | 64 ++++++++++++++++++++++------------------- 6 files changed, 42 insertions(+), 45 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 58d96534a..d662b632d 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -89,8 +89,7 @@ def with_rw_directory(func): @wraps(func) def wrapper(self): - path = tempfile.mktemp(prefix=func.__name__) - os.mkdir(path) + path = tempfile.mkdtemp(prefix=func.__name__) keep = False try: return func(self, path) diff --git a/test/performance/lib.py b/test/performance/lib.py index ceee6c2a1..d08e1027f 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -65,8 +65,7 @@ class TestBigRepoRW(TestBigRepoR): def setUp(self): self.gitrwrepo = None super().setUp() - dirname = tempfile.mktemp() - os.mkdir(dirname) + dirname = tempfile.mkdtemp() self.gitrwrepo = self.gitrorepo.clone(dirname, shared=True, bare=True, odbt=GitCmdObjectDB) self.puregitrwrepo = Repo(dirname, odbt=GitDB) diff --git a/test/test_base.py b/test/test_base.py index cdf82b74d..74f342071 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -68,12 +68,11 @@ def test_base_object(self): data = data_stream.read() assert data - tmpfilename = tempfile.mktemp(suffix="test-stream") - with open(tmpfilename, "wb+") as tmpfile: + with tempfile.NamedTemporaryFile(suffix="test-stream", delete=False) as tmpfile: self.assertEqual(item, item.stream_data(tmpfile)) tmpfile.seek(0) self.assertEqual(tmpfile.read(), data) - os.remove(tmpfilename) + os.remove(tmpfile.name) # Do it this way so we can inspect the file on failure. # END for each object type to create # Each has a unique sha. diff --git a/test/test_reflog.py b/test/test_reflog.py index 625466d40..1bd2e5dab 100644 --- a/test/test_reflog.py +++ b/test/test_reflog.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import os +import os.path as osp import tempfile from git.objects import IndexObject @@ -9,8 +9,6 @@ from test.lib import TestBase, fixture_path from git.util import Actor, rmtree, hex_to_bin -import os.path as osp - class TestRefLog(TestBase): def test_reflogentry(self): @@ -35,8 +33,7 @@ def test_reflogentry(self): def test_base(self): rlp_head = fixture_path("reflog_HEAD") rlp_master = fixture_path("reflog_master") - tdir = tempfile.mktemp(suffix="test_reflogs") - os.mkdir(tdir) + tdir = tempfile.mkdtemp(suffix="test_reflogs") rlp_master_ro = RefLog.path(self.rorepo.head) assert osp.isfile(rlp_master_ro) diff --git a/test/test_repo.py b/test/test_repo.py index 007b8ecb0..465bb2574 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -667,11 +667,10 @@ def test_tag_to_full_tag_path(self): self.assertEqual(value_errors, []) def test_archive(self): - tmpfile = tempfile.mktemp(suffix="archive-test") - with open(tmpfile, "wb") as stream: + with tempfile.NamedTemporaryFile("wb", suffix="archive-test", delete=False) as stream: self.rorepo.archive(stream, "0.1.6", path="doc") assert stream.tell() - os.remove(tmpfile) + os.remove(stream.name) # Do it this way so we can inspect the file on failure. @mock.patch.object(Git, "_call_process") def test_should_display_blame_information(self, git): diff --git a/test/test_util.py b/test/test_util.py index 1cb255cfe..6616e1067 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -359,48 +359,52 @@ def test_it_should_dashify(self): self.assertEqual("foo", dashify("foo")) def test_lock_file(self): - my_file = tempfile.mktemp() - lock_file = LockFile(my_file) - assert not lock_file._has_lock() - # Release lock we don't have - fine. - lock_file._release_lock() + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + lock_file = LockFile(my_file) + assert not lock_file._has_lock() + # Release lock we don't have - fine. + lock_file._release_lock() - # Get lock. - lock_file._obtain_lock_or_raise() - assert lock_file._has_lock() + # Get lock. + lock_file._obtain_lock_or_raise() + assert lock_file._has_lock() - # Concurrent access. - other_lock_file = LockFile(my_file) - assert not other_lock_file._has_lock() - self.assertRaises(IOError, other_lock_file._obtain_lock_or_raise) + # Concurrent access. + other_lock_file = LockFile(my_file) + assert not other_lock_file._has_lock() + self.assertRaises(IOError, other_lock_file._obtain_lock_or_raise) - lock_file._release_lock() - assert not lock_file._has_lock() + lock_file._release_lock() + assert not lock_file._has_lock() - other_lock_file._obtain_lock_or_raise() - self.assertRaises(IOError, lock_file._obtain_lock_or_raise) + other_lock_file._obtain_lock_or_raise() + self.assertRaises(IOError, lock_file._obtain_lock_or_raise) - # Auto-release on destruction. - del other_lock_file - lock_file._obtain_lock_or_raise() - lock_file._release_lock() + # Auto-release on destruction. + del other_lock_file + lock_file._obtain_lock_or_raise() + lock_file._release_lock() def test_blocking_lock_file(self): - my_file = tempfile.mktemp() - lock_file = BlockingLockFile(my_file) - lock_file._obtain_lock() - - # Next one waits for the lock. - start = time.time() - wait_time = 0.1 - wait_lock = BlockingLockFile(my_file, 0.05, wait_time) - self.assertRaises(IOError, wait_lock._obtain_lock) - elapsed = time.time() - start + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + lock_file = BlockingLockFile(my_file) + lock_file._obtain_lock() + + # Next one waits for the lock. + start = time.time() + wait_time = 0.1 + wait_lock = BlockingLockFile(my_file, 0.05, wait_time) + self.assertRaises(IOError, wait_lock._obtain_lock) + elapsed = time.time() - start + extra_time = 0.02 if os.name == "nt" or sys.platform == "cygwin": extra_time *= 6 # Without this, we get indeterministic failures on Windows. elif sys.platform == "darwin": extra_time *= 18 # The situation on macOS is similar, but with more delay. + self.assertLess(elapsed, wait_time + extra_time) def test_user_id(self): From 9e86053c885a27360d9d4699fafaef5e995d6ec5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 22:30:51 -0500 Subject: [PATCH 0927/1790] Replace the one use of mktemp in the git module This makes two related changes to git.index.util.TemporaryFileSwap: - Replace mktemp with mkstemp and then immediately closing the file. This avoids two possible name clashes: the usual name clash where the file may be created by another process between when mktemp generates the name and when the file is created, and the problem that mktemp does not check for files that contain the generated name as a part. The latter is specific to the use here, where a string generated by mktemp was manually concatenated to the base filename. This addresses that by passing the filename as the prefix for mkstemp to use. - Replace os.remove with os.replace and simplify. This is made necessary on Windows by the file already existing, due to mkstemp creating it. Deleting the file and allowing it to be recreated would reproduce the mktemp race condition in full (obscured and with extra steps). But os.replace supports an existing target file on all platforms. It is now also used in __exit__, where it allows the Windows check for the file to be removed, and (in any OS) better expresses the intent of the code to human readers. Furthermore, because one of the "look before you leap" checks in __exit__ is removed, the minor race condition in cleaning up the file is somewhat decreased. A small amount of related refactoring is included. The interface is not changed, even where it could be simplified (by letting __exit__ return None), and resource acquisition remains done on construction rather than in __enter__, because changing those aspects of the design could break some existing uses. Although the use of mktemp replaced here was in the git module and not the test suite, its use was to generate filenames for use in a directory that would ordinarily be under the user's control, such that the ability to carry out typical mktemp-related attacks would already require the ability to achieve the same goals more easily and reliably. Although TemporaryFileSwap is a public class that may be used directly by applications, it is only useful for making a temporary file in the same directory as an existing file. Thus the intended benefits of this change are mainly to code quality and a slight improvement in robustness. --- git/index/util.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/git/index/util.py b/git/index/util.py index b1aaa58fd..1c3b1c4ad 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -3,6 +3,7 @@ """Index utilities.""" +import contextlib from functools import wraps import os import os.path as osp @@ -40,12 +41,10 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = file_path - self.tmp_file_path = str(self.file_path) + tempfile.mktemp("", "", "") - # It may be that the source does not exist. - try: - os.rename(self.file_path, self.tmp_file_path) - except OSError: - pass + fd, self.tmp_file_path = tempfile.mkstemp(prefix=self.file_path, dir="") + os.close(fd) + with contextlib.suppress(OSError): # It may be that the source does not exist. + os.replace(self.file_path, self.tmp_file_path) def __enter__(self) -> "TemporaryFileSwap": return self @@ -57,10 +56,7 @@ def __exit__( exc_tb: Optional[TracebackType], ) -> bool: if osp.isfile(self.tmp_file_path): - if os.name == "nt" and osp.exists(self.file_path): - os.remove(self.file_path) - os.rename(self.tmp_file_path, self.file_path) - + os.replace(self.tmp_file_path, self.file_path) return False From 83b7ec61a1596971d4834e3c93a51b6efdf9e766 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Dec 2023 03:40:28 -0500 Subject: [PATCH 0928/1790] Set up CodeQL This is an automatically generated CodeQL workflow for the project. It is not yet customized. --- .github/workflows/codeql.yml | 81 ++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..7f64e35cb --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,81 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '27 10 * * 3' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 58547d82fa36165f611563c6bc449fde54dd3b2e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Dec 2023 03:41:16 -0500 Subject: [PATCH 0929/1790] Customize CodeQL - Run CodeQL on all branches. - Don't install Python dependencies. - Use v4 of actions/checkout. --- .github/workflows/codeql.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7f64e35cb..0d4d81efd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,9 +13,7 @@ name: "CodeQL" on: push: - branches: [ "main" ] pull_request: - branches: [ "main" ] schedule: - cron: '27 10 * * 3' @@ -45,13 +43,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} + setup-python-dependencies: false # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. From a315725c68295a02f3d007409ea3101430543289 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Dec 2023 03:04:54 -0500 Subject: [PATCH 0930/1790] Replace use of mktemp This uses NamedTemporaryFile with delete=False to replace the one use of the deprecated mktemp function in smmap (reported in #41). This avoids the race condition inherent to mktemp, as the file is named and created together in a way that is effectively atomic. Because NamedTemporaryFile is not being used to automatically delete the file, it use and cleanup are unaffected by the change. --- smmap/test/lib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/smmap/test/lib.py b/smmap/test/lib.py index ca91ee914..b15b0ec6a 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -18,12 +18,12 @@ class FileCreator: def __init__(self, size, prefix=''): assert size, "Require size to be larger 0" - self._path = tempfile.mktemp(prefix=prefix) self._size = size - with open(self._path, "wb") as fp: - fp.seek(size - 1) - fp.write(b'1') + with tempfile.NamedTemporaryFile("wb", prefix=prefix, delete=False) as file: + self._path = file.name + file.seek(size - 1) + file.write(b'1') assert os.path.getsize(self.path) == size From 5aeb6e073ebe007d43695976eccdcca64568d025 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:39:28 +0000 Subject: [PATCH 0931/1790] Bump gitdb/ext/smmap from `256c5a2` to `04dd210` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `256c5a2` to `04dd210`. - [Commits](https://github.com/gitpython-developers/smmap/compare/256c5a21de2d14aca02c9689d7d63f78c4e0ef61...04dd2103ee6e0b7483889e5feda25053c6df2b52) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 256c5a21d..04dd2103e 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 256c5a21de2d14aca02c9689d7d63f78c4e0ef61 +Subproject commit 04dd2103ee6e0b7483889e5feda25053c6df2b52 From d46c70d8059f20cab655720ec5792d18b71daad0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:11:25 +0000 Subject: [PATCH 0932/1790] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0d4d81efd..ae5241898 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,7 +47,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} setup-python-dependencies: false @@ -62,7 +62,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -75,6 +75,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" From b12a54a2fc7e988667f95145833a0dc8402b084d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Dec 2023 19:51:34 -0500 Subject: [PATCH 0933/1790] Use Path.touch to create files for rmtree tests It's not necessary to use `write_bytes(b"")`, because pathlib.Path has `touch()`. --- test/test_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 6616e1067..9408a91a0 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -46,7 +46,7 @@ def permission_error_tmpdir(tmp_path): """Fixture to test permissions errors in situations where they are not overcome.""" td = tmp_path / "testdir" td.mkdir() - (td / "x").write_bytes(b"") + (td / "x").touch() # Set up PermissionError on Windows, where we can't delete read-only files. (td / "x").chmod(stat.S_IRUSR) @@ -73,7 +73,7 @@ def test_deletes_nested_dir_with_files(self, tmp_path): td / "s" / "y", td / "s" / "z", ): - f.write_bytes(b"") + f.touch() try: rmtree(td) @@ -95,7 +95,7 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): for d in td, td / "sub": d.mkdir() for f in td / "x", td / "sub" / "y": - f.write_bytes(b"") + f.touch() f.chmod(0) try: @@ -115,7 +115,7 @@ def test_avoids_changing_permissions_outside_tree(self, tmp_path): dir1 = tmp_path / "dir1" dir1.mkdir() - (dir1 / "file").write_bytes(b"") + (dir1 / "file").touch() (dir1 / "file").chmod(stat.S_IRUSR) old_mode = (dir1 / "file").stat().st_mode From 6a8ed70a6003c13800d908ab055038eb61ba4ce9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Dec 2023 20:00:56 -0500 Subject: [PATCH 0934/1790] Run test_env_vars_for_windows_tests only on Windows This skips the tests of how the HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS environment variables affect the same-named attributes of git.util, except when testing on Windows. These are parsed only to ever set a True value on Windows, but checking that this is the case is less important ever since git.util.rmtree was changed to not check HIDE_WINDOWS_KNOWN_ERRORS on other systems (and this is covered in other tests). Setting the variables to True on non-Windows systems would still have a bad effect on the tests themselves, some of which use them as skip or xfail conditions separate from the skipping logic in git.util.rmtree. However, this is effectively using them as part of the test suite (which they were initially meant for and which they may eventually go back to being, for #790), where they would not ordinarily have tests. The benefit and motivation for running these tests only on Windows is that the tests can be simplified, so that their parameter sets are no longer confusing. That change is also made here. --- test/test_util.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 9408a91a0..f3769088c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -207,24 +207,28 @@ def _run_parse(name, value): ) return ast.literal_eval(output) + @pytest.mark.skipif( + os.name != "nt", + reason="These environment variables are only used on Windows.", + ) @pytest.mark.parametrize( "env_var_value, expected_truth_value", [ - (None, os.name == "nt"), # True on Windows when the environment variable is unset. + (None, True), # When the environment variable is unset. ("", False), (" ", False), ("0", False), - ("1", os.name == "nt"), + ("1", True), ("false", False), - ("true", os.name == "nt"), + ("true", True), ("False", False), - ("True", os.name == "nt"), + ("True", True), ("no", False), - ("yes", os.name == "nt"), + ("yes", True), ("NO", False), - ("YES", os.name == "nt"), + ("YES", True), (" no ", False), - (" yes ", os.name == "nt"), + (" yes ", True), ], ) @pytest.mark.parametrize( From 487a4fdb38efc82d30111e49f941a25cba4aa7a7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 22:39:11 -0500 Subject: [PATCH 0935/1790] Add a test for git.index.util.TemporaryFileSwap This is a general test for TemporaryFileSwap, but by being parametrized by the type of file_path, it reveals a regression introduced in 9e86053 (#1770). TemporaryFileSwap still works when file_path is a string, but is now broken when it is a Path. That worked before, and the type annotations document that it should be able to work. This is at least a bug because TemporaryFileSwap is public. (I am unsure whether, in practice, GitPython itself uses it in a way that sometimes passes a Path object as file_path. But code that uses GitPython may call it directly and pass Path.) --- test/test_index.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/test_index.py b/test/test_index.py index 2f97f0af8..c3f3b4fae 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -34,10 +34,11 @@ ) from git.index.fun import hook_path from git.index.typ import BaseIndexEntry, IndexEntry +from git.index.util import TemporaryFileSwap from git.objects import Blob -from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo from git.util import Actor, hex_to_bin, rmtree from gitdb.base import IStream +from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo HOOKS_SHEBANG = "#!/usr/bin/env sh\n" @@ -1087,3 +1088,25 @@ def test_index_add_pathlike(self, rw_repo): file.touch() rw_repo.index.add(file) + + +class TestIndexUtils: + @pytest.mark.parametrize("file_path_type", [str, Path]) + def test_temporary_file_swap(self, tmp_path, file_path_type): + file_path = tmp_path / "foo" + file_path.write_bytes(b"some data") + + with TemporaryFileSwap(file_path_type(file_path)) as ctx: + assert Path(ctx.file_path) == file_path + assert not file_path.exists() + + # Recreate it with new data, so we can observe that they're really separate. + file_path.write_bytes(b"other data") + + temp_file_path = Path(ctx.tmp_file_path) + assert temp_file_path.parent == file_path.parent + assert temp_file_path.name.startswith(file_path.name) + assert temp_file_path.read_bytes() == b"some data" + + assert not temp_file_path.exists() + assert file_path.read_bytes() == b"some data" # Not b"other data". From 1ddf953ea0d7625485ed02c90b03aae7f939ec46 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 22:58:17 -0500 Subject: [PATCH 0936/1790] Fix TemporaryFileSwap bug when file_path is a Path This fixes the regression introduced in 9e86053 (#1770) where the file_path argument to TemporaryFileSwap.__init__ could no longer be a Path object. The change also makes this truer to the code from before #1770, still without the race condition fixed there, in that str was called on file_path then as well. However, it is not clear that this is a good thing, because this is not an idiomatic use of mkstemp. The reason the `prefix` cannot be a Path is that it is expected to be a filename prefix, with leading directories given in the `dir` argument. --- git/index/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/util.py b/git/index/util.py index 1c3b1c4ad..03de80f71 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -41,7 +41,7 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = file_path - fd, self.tmp_file_path = tempfile.mkstemp(prefix=self.file_path, dir="") + fd, self.tmp_file_path = tempfile.mkstemp(prefix=str(self.file_path), dir="") os.close(fd) with contextlib.suppress(OSError): # It may be that the source does not exist. os.replace(self.file_path, self.tmp_file_path) From b438c459d91243b685f540ccb1c124260e9d28b9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 23:29:14 -0500 Subject: [PATCH 0937/1790] Refactor TemporaryFileSwap.__init__ for clarity This changes the mkstemp call TemporaryFileSwap uses to atomically create (and thus reserve) the temporary file, so that it passes only a filename (i.e., basename) prefix as `prefix`, and passes all other path components (i.e., the directory) as `dir`. (If the original path has no directory, then `dir` is "" as before.) This makes the mkstemp call *slightly* more idiomatic. This also makes it clearer, as it is no longer using `prefix` for something that feels like it should be possible to pass as a Path object. Although mkstemp does not accept a Path as `prefix`, it does (as expected) accept one as `dir`. However, to keep the code simple, I'm passing str for both. The os.path.split function accepts both str and Path (since Python 3.6), and returns str objects, which are now used for the `dir` and `prefix` arguments to mkstemp. For unusual cases, this may technically not be a refactoring. For example, a file_path of "a/b//c" will be split into "a/b" and "c". If the automatically generated temporary file suffix is "xyz", then that results in a tmp_file_path of "a/b/cxyz" where "a/b//cxyz" would have been used before. The tmp_file_path attribute of a TemporaryFileSwap object is public (and used in filesystem calls). However, no guarantee has ever been given that the temporary file path have the original path as an exact string prefix. I believe the slightly weaker relationship I expressed in the recently introduced test_temporary_file_swap -- another file in the same directory, named with the original filename with more characters, consisting of equivalent path components in the same order -- has always been the intended one. Note that this slight possible variation does not apply to the file_path attribute. That attribute is always kept exactly as it was, both in its type and its value, and it always used unmodified in calls that access the filesystem. --- git/index/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/index/util.py b/git/index/util.py index 03de80f71..61039fe7c 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -41,7 +41,8 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = file_path - fd, self.tmp_file_path = tempfile.mkstemp(prefix=str(self.file_path), dir="") + dirname, basename = osp.split(file_path) + fd, self.tmp_file_path = tempfile.mkstemp(prefix=basename, dir=dirname) os.close(fd) with contextlib.suppress(OSError): # It may be that the source does not exist. os.replace(self.file_path, self.tmp_file_path) From 4e91a6c7b521dc7d0331dbb5455e9c424d26d655 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 23:56:55 -0500 Subject: [PATCH 0938/1790] Tweak formatting for `@pytest.mark.parametrize` This removes the "fmt: off" / "fmt: on" directives around the `@pytest.mark.parametrize` decoration on test_blob_filter, and reformats it with black, for consistency with other such decorations. The style used there, *if* it could be specified as a rule and thus used without "fmt:" directives, may be nicer than how black formats multi-line mark decorations. However, since that decoration was written, there have been a number of other such decorations, which have been in black style. This also removes the only (or only remaining?) "fmt:" directive in the codebase. As such, it should possibly have been done in #1760. --- test/test_blob_filter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/test_blob_filter.py b/test/test_blob_filter.py index a91f211bf..ddd83079a 100644 --- a/test/test_blob_filter.py +++ b/test/test_blob_filter.py @@ -14,14 +14,15 @@ from git.types import PathLike -# fmt: off -@pytest.mark.parametrize('paths, path, expected_result', [ - ((Path("foo"),), Path("foo"), True), - ((Path("foo"),), Path("foo/bar"), True), - ((Path("foo/bar"),), Path("foo"), False), - ((Path("foo"), Path("bar")), Path("foo"), True), -]) -# fmt: on +@pytest.mark.parametrize( + "paths, path, expected_result", + [ + ((Path("foo"),), Path("foo"), True), + ((Path("foo"),), Path("foo/bar"), True), + ((Path("foo/bar"),), Path("foo"), False), + ((Path("foo"), Path("bar")), Path("foo"), True), + ], +) def test_blob_filter(paths: Sequence[PathLike], path: PathLike, expected_result: bool) -> None: """Test the blob filter.""" blob_filter = BlobFilter(paths) From f664a0b85dd6be4bb8fd16d7283fd1be0f81c8c8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Dec 2023 00:58:33 -0500 Subject: [PATCH 0939/1790] Add xfail marks to hook tests for WinBashStatus.Absent Precise xfail marks were added to commit hook tests in #1745, but in a few tests I didn't get it right for WinBashStatus.Absent. That is, on a Windows system that has no bash.exe at all: - More of the tests are unable to pass than have xfail marks. - One of the tests, test_commit_msg_hook_success, which does have an xfail mark for that, wrongly combines it with the xfail mark for WinBashStatus.Wsl. That test is the only one where the WSL bash.exe, even when a working WSL distribution is installed, is unable to pass. But using a single mark there is wrong, in part because the "reason" is not correct for both, but even more so because the exceptions they raise are different: AssertionError is raised when the WSL bash.exe is used in that test, but when bash.exe is altogether absent, HookExecutionError is raised. This fixes that by adding xfail marks for WinBashStatus.Absent where missing, and splitting test_commit_msg_hook_success's xfail mark that unsuccessfully tried to cover two conditions into two separate marks, each of which gives a correct reason and exception. This commit also rewords the xfail reason given for WslNoDistro, which was somewhat unclear and potentially ambiguous, to make it clearer. --- test/test_index.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 2f97f0af8..3f0990ecc 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -991,9 +991,14 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + rasies=HookExecutionError, + ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=HookExecutionError, ) @with_rw_repo("HEAD", bare=True) @@ -1004,7 +1009,7 @@ def test_pre_commit_hook_success(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=AssertionError, ) @with_rw_repo("HEAD", bare=True) @@ -1030,13 +1035,18 @@ def test_pre_commit_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @pytest.mark.xfail( - type(_win_bash_status) in {WinBashStatus.Absent, WinBashStatus.Wsl}, + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + rasies=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Wsl, reason="Specifically seems to fail on WSL bash (in spite of #1399)", raises=AssertionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=HookExecutionError, ) @with_rw_repo("HEAD", bare=True) @@ -1054,7 +1064,7 @@ def test_commit_msg_hook_success(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=AssertionError, ) @with_rw_repo("HEAD", bare=True) From e1486474a2381763e11932b3bb15e08f6e9faf53 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Dec 2023 01:31:54 -0500 Subject: [PATCH 0940/1790] Add a direct test of run_commit_hook This has three benefits: - run_commit_hook is public, being listed in git.index.fun.__all__, so it makes sense for it to have its own test. - When investigating (future, or current xfail-covered) failure of functions that use run_commit_hook, it will be useful to compare the results of other tests that already do exist to that of a direct test of run_commit_hook. - When changing which bash.exe run_commit_hook selects to use on Windows (including to ameliorate the limitation covered by the WinBashStatus.WslNoDistro xfail marks, or to attempt other possible changes suggested in #1745), or even just to investigate the possibility of doing so, it will make sense to add tests like this test but for more specific conditions or edge cases. Having this typical-case test to compare to should be helpful both for writing such tests and for efficiently verifying that the conditions they test are really the triggers for any failures. --- test/test_index.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/test_index.py b/test/test_index.py index 3f0990ecc..afceda131 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -32,7 +32,7 @@ InvalidGitRepositoryError, UnmergedEntriesError, ) -from git.index.fun import hook_path +from git.index.fun import hook_path, run_commit_hook from git.index.typ import BaseIndexEntry, IndexEntry from git.objects import Blob from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo @@ -991,6 +991,24 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + rasies=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.WslNoDistro, + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", + raises=HookExecutionError, + ) + @with_rw_repo("HEAD", bare=True) + def test_run_commit_hook(self, rw_repo): + index = rw_repo.index + _make_hook(index.repo.git_dir, "fake-hook", "echo 'ran fake hook' >output.txt") + run_commit_hook("fake-hook", index) + output = Path(rw_repo.git_dir, "output.txt").read_text(encoding="utf-8") + self.assertEqual(output, "ran fake hook\n") + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", From 6e4cee4fa708465cf714e36a9dc8b9b6b94acc0c Mon Sep 17 00:00:00 2001 From: Stefan Gmeiner Date: Thu, 21 Dec 2023 21:33:37 +0100 Subject: [PATCH 0941/1790] Fix Items of type PathLike --- git/index/base.py | 2 +- test/test_index.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 112ad3feb..cffb213a9 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -940,7 +940,7 @@ def _items_to_rela_paths( for item in items: if isinstance(item, (BaseIndexEntry, (Blob, Submodule))): paths.append(self._to_relative_path(item.path)) - elif isinstance(item, str): + elif isinstance(item, (str, os.PathLike)): paths.append(self._to_relative_path(item)) else: raise TypeError("Invalid item type: %r" % item) diff --git a/test/test_index.py b/test/test_index.py index 6b746b8b4..50d941e83 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -558,14 +558,16 @@ def test_index_mutation(self, rw_repo): def mixed_iterator(): count = 0 for entry in index.entries.values(): - type_id = count % 4 - if type_id == 0: # path + type_id = count % 5 + if type_id == 0: # path (str) yield entry.path - elif type_id == 1: # blob + elif type_id == 1: # path (PathLike) + yield Path(entry.path) + elif type_id == 2: # blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 2: # BaseIndexEntry + elif type_id == 3: # BaseIndexEntry yield BaseIndexEntry(entry[:4]) - elif type_id == 3: # IndexEntry + elif type_id == 4: # IndexEntry yield entry else: raise AssertionError("Invalid Type") From 53e73830f64e13a88b0e246fb825944a6fe2848e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 00:57:14 -0500 Subject: [PATCH 0942/1790] Remove explicit PushInfo/FetchInfo inheritance from object I had missed these in f78587f (#1725). --- git/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index 4055dba2e..98a421b3a 100644 --- a/git/remote.py +++ b/git/remote.py @@ -130,7 +130,7 @@ def to_progress_instance( return progress -class PushInfo(IterableObj, object): +class PushInfo(IterableObj): """ Carries information about the result of a push operation of a single head:: @@ -300,7 +300,7 @@ def raise_if_error(self) -> None: raise self.error -class FetchInfo(IterableObj, object): +class FetchInfo(IterableObj): """ Carries information about the results of a fetch operation of a single head:: From 96fc3547cf62c5ade1477c24566c8b34254a1507 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 01:55:30 -0500 Subject: [PATCH 0943/1790] Add tests for current Submodule.iter_items behavior Where the behavior is intended. In the case of an invalid hash (or IOError, which in Python 2 was a subclass of OSError but now is just another name for it), the behavior of just yielding no items may be unintuitive, since on most other errors an exception is raised. However, examining the code reveals this behavior is clearly intentional. Furthrmore, it may be reasonable for applications to rely on it, and it may be convenient in some situations. For backward compatibility, it probably can't be changed significantly. This adds tests that show both an error that does raise an error-representing exception -- a well-formed hash not present in the repository raising ValueError with a suitable message -- and an error that silently causes the iterator to yield zero items. --- test/test_submodule.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index 4dc89f98f..5226d7a6e 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -688,6 +688,17 @@ def test_root_module(self, rwrepo): # gitdb: has either 1 or 2 submodules depending on the version. assert len(nsm.children()) >= 1 and nsmc.module_exists() + def test_iter_items_from_nonexistent_hash(self): + it = Submodule.iter_items(self.rorepo, "b4ecbfaa90c8be6ed6d9fb4e57cc824663ae15b4") + with self.assertRaisesRegex(ValueError, r"\bcould not be resolved\b"): + next(it) + + def test_iter_items_from_invalid_hash(self): + """Check legacy behavaior on BadName (also applies to IOError, i.e. OSError).""" + it = Submodule.iter_items(self.rorepo, "xyz") + with self.assertRaises(StopIteration): + next(it) + @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): assert len(list(rwrepo.iter_submodules())) == 0 From f5dc1c4713dfb937d45d10a595ac879d6e76481c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 02:16:28 -0500 Subject: [PATCH 0944/1790] Expand "invalid hash" test to assert normal StopIteration Returning an explicit value from a generator function causes that value to be bound to the `value` attribute of the StopIteration exception. This is available as the result of "yield from" when it is used as an expression; or by explicitly catching StopIteration, binding the StopIteration exception to a variable, and accessing the attribute. This feature of generators is rarely used. The `return iter([])` statement in Submodule.iter_items uses this feature, causing the resulting StopIteration exception object to have a `value` attribute that refers to a separate second iterator that also yields no values (#1779). From context, this behavior is clearly not the goal; a bare return statement should be used here (which has the same effect except for the `value` attribute of the StopIteration exception). The code had used a bare return prior to 82b131c (#1282), when `return` was changed to `return iter([])`. That was part of a change that added numerous type annotations. It looks like it was either a mistake, or possibly an attempt to work around an old bug in a static type checker. This commit extends the test_iter_items_from_invalid_hash test to assert that the `value` attribute of the StopIteration is its usual default value of None. This commit only extends the test; it does not fix the bug. --- test/test_submodule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 5226d7a6e..993f6b57e 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -696,8 +696,9 @@ def test_iter_items_from_nonexistent_hash(self): def test_iter_items_from_invalid_hash(self): """Check legacy behavaior on BadName (also applies to IOError, i.e. OSError).""" it = Submodule.iter_items(self.rorepo, "xyz") - with self.assertRaises(StopIteration): + with self.assertRaises(StopIteration) as ctx: next(it) + self.assertIsNone(ctx.exception.value) @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): From c3c008c4971f9d4189dc08e88334a207ce14298c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 02:44:56 -0500 Subject: [PATCH 0945/1790] In Submodule.iter_items, don't attach second empty iterator This fixes the minor bug where a separate empty iterator was bound to the StopIteration exception raised as a result of returning from the generator function (#1779). This change does not cause what exceptions are raised from GitPython code in any situations, nor how many items any iterators yield. --- git/objects/submodule/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 651d9535c..49dfedf9a 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1401,7 +1401,7 @@ def iter_items( pc = repo.commit(parent_commit) # Parent commit instance parser = cls._config_parser(repo, pc, read_only=True) except (IOError, BadName): - return iter([]) + return # END handle empty iterator for sms in parser.sections(): From dfee31f2100d7f4a653c69c4a5a505607fe328e1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 03:48:57 -0500 Subject: [PATCH 0946/1790] Improve self-documentation of IterableObj and related classes - Fill in the missing part of the explanation of why to favor iter_items over list_items in IterableObj and Iterable (#1775). - Move the explanation of how subclasses must treat arguments from the list_items methods to the iter_items methods, because the iter_items methdos are the ones that are abstract and must be overridden by a well-behaved subclass, and also because, since the iter_items methods are preferred for use, they should be the place where less duplicated shared documentation resides. - Subtantially reword the existing documentation for clarity, especially regarding the signifance of extra args and kwargs. - Put the iter_items method before (i.e. above) the list_items method (in each of the IterableObj and Iterable classes), because that method is the one that should be used more often, and because it is also now the one with the more detailed docstring. - Remove and old comment on a return type that said exactly the exact same thing as the annotation. - In Iterable, note deprecation more consistently (and thus in more places). - Rewrite the IterableClassWatcher class docstring to explain exactly what that metaclass achieves. --- git/util.py | 84 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/git/util.py b/git/util.py index 0a5da7d71..5acc001f7 100644 --- a/git/util.py +++ b/git/util.py @@ -1183,7 +1183,8 @@ def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: class IterableClassWatcher(type): - """Metaclass that watches.""" + """Metaclass that issues :class:`DeprecationWarning` when :class:`git.util.Iterable` + is subclassed.""" def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: for base in bases: @@ -1199,23 +1200,42 @@ def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: class Iterable(metaclass=IterableClassWatcher): - """Defines an interface for iterable items, so there is a uniform way to retrieve - and iterate items within the git repository.""" + """Deprecated, use :class:`IterableObj` instead. + + Defines an interface for iterable items, so there is a uniform way to retrieve + and iterate items within the git repository. + """ __slots__ = () _id_attribute_ = "attribute that most suitably identifies your instance" @classmethod - def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: + def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: + # return typed to be compatible with subtypes e.g. Remote + """Deprecated, use :class:`IterableObj` instead. + + Find (all) items of this type. + + Subclasses can specify ``args`` and ``kwargs`` differently, and may use them for + filtering. However, when the method is called with no additional positional or + keyword arguments, subclasses are obliged to to yield all items. + + :return: Iterator yielding Items """ - Deprecated, use IterableObj instead. + raise NotImplementedError("To be implemented by Subclass") + + @classmethod + def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: + """Deprecated, use :class:`IterableObj` instead. + + Find (all) items of this type and collect them into a list. - Find all items of this type - subclasses can specify args and kwargs differently. - If no args are given, subclasses are obliged to return all items if no additional - arguments arg given. + For more information about the arguments, see :meth:`list_items`. - :note: Favor the iter_items method as it will + :note: Favor the :meth:`iter_items` method as it will avoid eagerly collecting + all items. When there are many items, that can slow performance and increase + memory usage. :return: list(Item,...) list of item instances """ @@ -1223,15 +1243,6 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list - @classmethod - def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: - # return typed to be compatible with subtypes e.g. Remote - """For more information about the arguments, see list_items. - - :return: Iterator yielding Items - """ - raise NotImplementedError("To be implemented by Subclass") - @runtime_checkable class IterableObj(Protocol): @@ -1246,13 +1257,30 @@ class IterableObj(Protocol): _id_attribute_: str @classmethod - def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_IterableObj]: + @abstractmethod + def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_IterableObj]: + # Return-typed to be compatible with subtypes e.g. Remote. + """Find (all) items of this type. + + Subclasses can specify ``args`` and ``kwargs`` differently, and may use them for + filtering. However, when the method is called with no additional positional or + keyword arguments, subclasses are obliged to to yield all items. + + For more information about the arguments, see list_items. + + :return: Iterator yielding Items """ - Find all items of this type - subclasses can specify args and kwargs differently. - If no args are given, subclasses are obliged to return all items if no additional - arguments arg given. + raise NotImplementedError("To be implemented by Subclass") + + @classmethod + def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_IterableObj]: + """Find (all) items of this type and collect them into a list. + + For more information about the arguments, see :meth:`list_items`. - :note: Favor the iter_items method as it will + :note: Favor the :meth:`iter_items` method as it will avoid eagerly collecting + all items. When there are many items, that can slow performance and increase + memory usage. :return: list(Item,...) list of item instances """ @@ -1260,16 +1288,6 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list - @classmethod - @abstractmethod - def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_IterableObj]: # Iterator[T_IterableObj]: - # Return-typed to be compatible with subtypes e.g. Remote. - """For more information about the arguments, see list_items. - - :return: Iterator yielding Items - """ - raise NotImplementedError("To be implemented by Subclass") - # } END classes From 94a85d1159e6374e901df4e3bd284cf55b623e66 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 14:31:28 -0500 Subject: [PATCH 0947/1790] Convert constant and attribute comments to docstrings These are "non-reified docstrings" as described in #1734. They are not made part of the data model, but most editors will display them, including on symbols present in code of other projects that use the GitPython library. A number of these have already been added, but this adds what will probably be most of the remaining ones. For the most part, this doesn't create documentation where there wasn't any, but instead converts existing comments to "docstrings." In a handful of cases, they are expanded, reworded, or a docstring added. This also fixes some small style inconsistencies that were missed in #1725, and moves a comment that had become inadvertently displaced due to autoformatting to the item it was meant for. The major omission here is HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS. This doesn't convert the comments above them to "docstrings," for a few reasons. They are not specific to either of the symbols, they are oriented toward considerations that are not really relevant except when developing GitPython itself, and they are out of date. Also, because HIDE_WINDOWS_KNOWN_ERRORS is listed in __all__, increasing the level of documentation for it might be taken as a committment to preserve some aspect of its current behavior, which could interfere with progress on #790. So I've kept those comments as comments, and unchanged, for now. --- git/cmd.py | 25 ++++++++++++++++--------- git/config.py | 20 ++++++++++++-------- git/diff.py | 2 +- git/index/base.py | 6 ++++-- git/index/fun.py | 4 +++- git/objects/submodule/base.py | 11 ++++++----- git/repo/base.py | 31 ++++++++++++++++++++----------- git/util.py | 4 ++-- 8 files changed, 64 insertions(+), 39 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 27148d3d6..9518c2c8c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -273,23 +273,30 @@ def __setstate__(self, d: Dict[str, Any]) -> None: # CONFIGURATION - git_exec_name = "git" # Default that should work on Linux and Windows. + git_exec_name = "git" + """Default git command that should work on Linux, Windows, and other systems.""" - # Enables debugging of GitPython's git commands. GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) + """Enables debugging of GitPython's git commands.""" - # If True, a shell will be used when executing git commands. - # This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 - # and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. - # Override this value using `Git.USE_SHELL = True`. USE_SHELL = False + """If True, a shell will be used when executing git commands. + + This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 + and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. + + Override this value using ``Git.USE_SHELL = True``. + """ - # Provide the full path to the git executable. Otherwise it assumes git is in the path. _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" + GIT_PYTHON_GIT_EXECUTABLE = None - # Note that the git executable is actually found during the refresh step in - # the top level __init__. + """Provide the full path to the git executable. Otherwise it assumes git is in the path. + + Note that the git executable is actually found during the refresh step in + the top level ``__init__``. + """ @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: diff --git a/git/config.py b/git/config.py index 5708a7385..2730ddaf3 100644 --- a/git/config.py +++ b/git/config.py @@ -65,12 +65,14 @@ log.addHandler(logging.NullHandler()) -# The configuration level of a configuration file. CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") +"""The configuration level of a configuration file.""" -# Section pattern to detect conditional includes. -# https://git-scm.com/docs/git-config#_conditional_includes CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") +"""Section pattern to detect conditional includes. + +See: https://git-scm.com/docs/git-config#_conditional_includes +""" class MetaParserBuilder(abc.ABCMeta): # noqa: B024 @@ -283,12 +285,14 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): """ # { Configuration - # The lock type determines the type of lock to use in new configuration readers. - # They must be compatible to the LockFile interface. - # A suitable alternative would be the BlockingLockFile t_lock = LockFile - re_comment = re.compile(r"^\s*[#;]") + """The lock type determines the type of lock to use in new configuration readers. + They must be compatible to the LockFile interface. + A suitable alternative would be the :class:`~git.util.BlockingLockFile`. + """ + + re_comment = re.compile(r"^\s*[#;]") # } END configuration optvalueonly_source = r"\s*(?P