calculate
/
calculate-utils-3-lib
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Оптимизация работы с git

Browse Source
master3.4 3.4.5.2
Mike Hiretsky 9 years ago
parent 0e657ae22d
commit b7f3c7359f
2 changed files with 239 additions and 20 deletions
Show Stats Download Patch File Download Diff File

237
pym/calculate/lib/utils/git.py
Unescape View File

@ -16,6 +16,7 @@
import re
import sys
import os
from os import path
from calculate.lib.configparser import ConfigParser
import shutil
@ -24,6 +25,7 @@ from calculate.lib.utils.files import (getProgPath, STDOUT,
PercentProgress, process, readFile,
writeFile, listDirectory, removeDir)
from calculate.lib.utils.tools import AddonError
from calculate.lib.utils.ip import check_port
_ = lambda x: x
from calculate.lib.cl_lang import setLocalTranslate
@ -152,7 +154,27 @@ class Git:
return d
return {}
def cloneTagRepository(self, url, rpath, reference, cb_progress=None):
def _copy_index(self, git_dir, file_url):
"""
Переместить index файл из одного репозитория в другой
:param git_dir:
:param file_url:
:return:
"""
old_index = path.join(file_url[7:], "index")
new_index = path.join(git_dir, "index")
if path.exists(old_index):
try:
with open(old_index, 'r') as in_f:
with writeFile(new_index) as out_f:
out_f.write(in_f.read())
return True
except IOError:
pass
return False
def cloneTagRepository(self, url, rpath, reference, cb_progress=None,
copy_index=False):
"""
Сделать локальную копию репозитория для переключения по тэгам
@ -170,21 +192,49 @@ class Git:
error = []
command = self._git_command(git_dir, rpath, error, cb_progress)
# получить последние коммиты из удаленного репозитория
if command("init"):
if not command("fetch", "--depth=1", url, reference, part=4,
end=False):
def command_fetch(**kw):
"""
Получить данные репозитория (если shallow.lock - удалит и повторить)
"""
res = command("fetch", "--depth=1", url, reference, **kw)
if not res:
if self._clear_lock(git_dir, "shallow.lock", error):
res = command("fetch", "--depth=1", url, reference, **kw)
if not res:
raise GitError(
_("Reference {mark} not found in repository {url}"
).format(mark=reference, url=url))
if command("reset", "--hard", "FETCH_HEAD", part=4, startpart=3):
if self.saveFetchHead(git_dir) == self.Reference.Branch:
command("symbolic-ref", "HEAD", "refs/heads/%s" % reference)
return res
# получить последние коммиты из удаленного репозитория
if command("init"):
if copy_index:
copy_index = self._copy_index(git_dir, url)
if copy_index:
command_fetch(part=3)
fetch_type = self.saveFetchHead(git_dir)
if fetch_type == self.Reference.Tag:
# создать master ветку с ссылкой на HEAD
self.saveFetchHead(
git_dir, force_type=self.Reference.Branch,
force_ref=self.Reference.Master)
reference = self.Reference.Master
command("symbolic-ref", "HEAD", "refs/heads/%s" % reference)
return True
else:
command_fetch(part=4, end=False)
if command("reset", "--hard", "FETCH_HEAD", part=4,
startpart=3):
fetch_type = self.saveFetchHead(git_dir)
if fetch_type == self.Reference.Branch:
command("symbolic-ref", "HEAD",
"refs/heads/%s" % reference)
return True
raise GitError(_("Failed to clone repository {url}").format(
url=url), error[-1])
# progressParam = {'fetch': {'part': 4, 'end': False},
# 'checkout': {'part': 4, 'startpart': 3}}
# 'checkout': {'part': 4, 'startpart': 3}}
def trimRepository(self, rpath, cb_progress=None):
"""
@ -196,7 +246,6 @@ class Git:
return False
reference = head_info['reference']
print reference
git_dir_old = "%s.old" % git_dir
try:
shutil.move(git_dir, git_dir_old)
@ -204,7 +253,8 @@ class Git:
raise GitError(_("Failed to optimize repository %s") % rpath)
self.cloneTagRepository("file://%s" % git_dir_old,
rpath, reference, cb_progress)
rpath, reference, cb_progress,
copy_index=True)
try:
removeDir(git_dir_old)
except Exception:
@ -245,7 +295,19 @@ class Git:
return self.Reference.Branch
return self.Reference.Unknown
def updateTagRepository(self, url, rpath, reference, cb_progress=None):
def _clear_lock(self, git_dir, lock_fn, error):
if lock_fn in error[-1]:
fn = path.join(git_dir, lock_fn)
if path.exists(fn):
try:
os.unlink(fn)
return True
except OSError:
pass
return False
def updateTagRepository(self, url, rpath, reference, cb_progress=None,
clean=False):
"""
Обновить данные до указанного тега репозитория
@ -264,19 +326,33 @@ class Git:
try:
if (self.reference_type(rpath, reference) == Git.Reference.Tag and
self.isTagRepository(rpath) and
self.getCurrentCommit(rpath) == self.getCommit(rpath,
reference)):
status = self.getStatusInfo(rpath)
if status and not status["files"]:
self.getCurrentCommit(rpath) == self.getCommit(
rpath, reference)):
if clean:
status = self.getStatusInfo(rpath)
if status and not status["files"]:
return False
else:
return False
except GitError:
pass
if not command("fetch", "--depth=1", url, reference, part=4, end=False):
res = command("fetch", "--depth=1", url, reference, part=4, end=False)
if not res:
if self._clear_lock(git_dir, "shallow.lock", error):
res = command("fetch", "--depth=1",
url, reference, part=4, end=False)
if not res:
raise GitError(
_("Reference {mark} not found in repository {url}"
).format(mark=reference, url=url))
if command("reset", "--hard", "FETCH_HEAD", part=4, startpart=3):
command("clean", "-df")
res = command("reset", "--hard", "FETCH_HEAD", part=4, startpart=3)
if not res:
if self._clear_lock(git_dir, "index.lock", error):
res = command("reset", "--hard", "FETCH_HEAD",
part=4, startpart=3)
if res:
if clean:
command("clean", "-df")
if self.saveFetchHead(git_dir) == self.Reference.Branch:
command("symbolic-ref", "HEAD", "refs/heads/%s" % reference)
return True
@ -285,13 +361,17 @@ class Git:
# TODO: метод удаление лишних объектов, оставляя только текущий коммит
def saveFetchHead(self, git_dir):
def saveFetchHead(self, git_dir, force_type=None, force_ref=None):
"""
Сохранить FEACH_HEAD как объект (tag или branch)
:param git_dir: каталог .git репозитория
:return: Reference
"""
fetch_info = self._getFetchHeadInfo(git_dir)
if force_type is not None:
fetch_info['type'] = force_type
if force_ref is not None:
fetch_info['reference'] = force_ref
tag_file = None
if fetch_info.get('type', '') == self.Reference.Tag:
tag_file = path.join(git_dir, "refs/tags/%s"
@ -305,6 +385,91 @@ class Git:
return fetch_info['type']
return self.Reference.Unknown
class GitProtocol(object):
SSH = 22
Git = 9418
HTTP = 80
HTTPS = 443
FTP = 21
FTPS = 990
Rsync = 873
Unknown = 0
def _parse_url(self, url):
"""
Разобрать url на составляющие: тип и хост
:param url: разбираемый URL
:return: (тип, хост)
"""
net_protocols = {
'ssh': self.GitProtocol.SSH,
'ssh+git': self.GitProtocol.SSH,
'git+ssh': self.GitProtocol.SSH,
'http': self.GitProtocol.HTTP,
'https': self.GitProtocol.HTTPS,
'ftp': self.GitProtocol.FTP,
'ftps': self.GitProtocol.FTPS,
'rsync': self.GitProtocol.Rsync,
'git': self.GitProtocol.Git}
re_host = re.compile(
"(%s)://(?:[^@]+@)?([^:/]+)" %
"|".join(x.replace("+", r"\+") for x in net_protocols.keys())
)
net_match = re_host.search(url)
if net_match:
return net_protocols[net_match.group(1)], net_match.group(2)
re_scp = re.compile("(?:[^@]+@)?([^:]+):(//)?")
scp_match = re_scp.search(url)
if scp_match and scp_match.group(2) is None:
return self.GitProtocol.SSH, scp_match.group(1)
return self.GitProtocol.Unknown, ""
def checkUrl(self, url):
"""
Проверить доступность указанного URL
ssh://[user@]host.xz[:port]/path/to/repo.git/
git://host.xz[:port]/path/to/repo.git/
http[s]://host.xz[:port]/path/to/repo.git/
ftp[s]://host.xz[:port]/path/to/repo.git/
rsync://host.xz/path/to/repo.git/
[user@]host.xz:path/to/repo.git/
/path/to/repo.git/
file:///path/to/repo.git/
:param url: проверяемый URL
:return:
"""
git_port, target = self._parse_url(url)
if git_port != self.GitProtocol.Unknown:
return check_port(target, git_port)
return True
def isNeedUnpack(self, rpath, fns=(".gitignore",
"metadata/.gitignore")):
"""
Проверить репозиторий на то, что он еще не распакован
:param rpath: путь до репозитория
:param fns: имя файла
:return:
"""
for fn in fns:
if path.exists(path.join(rpath, fn)):
return False
else:
git_dir = self._gitDir(rpath)
git_log = process(self._git, "--git-dir", git_dir, "log",
"-n1", "--format=format:%H",
"--quiet", "--", *fns, stderr=STDOUT)
if git_log.success() and git_log.read().strip():
return True
status = self.getStatusInfo(rpath)
if not status or status['files']:
return True
return False
def cloneRevRepository(self, url, rpath, branch, revision,
cb_progress=None):
"""
@ -586,3 +751,35 @@ class Git:
"repository").format(branch=branch,
rpath=rpath), error)
return True
class MTimeKeeper(object):
"""
Сохранить и восстановить mtime на файлы
"""
def __init__(self, dn):
self.data = {}
self.dn = dn
def save(self):
for fn in listDirectory(self.dn, fullPath=True):
try:
stat_data = os.stat(fn)
self.data[fn] = {'size': stat_data.st_size,
'mtime': stat_data.st_mtime,
'atime': stat_data.st_atime,
}
except OSError:
pass
def restore(self):
for fn in self.data.keys():
try:
if path.exists(fn):
stat_data = os.stat(fn)
if stat_data.st_size == self.data[fn]['size']:
os.utime(fn, (self.data[fn]['atime'],
self.data[fn]['mtime']))
except OSError:
pass

22
pym/calculate/lib/utils/ip.py
Unescape View File

@ -473,3 +473,25 @@ class Pinger(object):
timeLeft = timeLeft - howLongInSelect
if timeLeft <= 0:
return None, 0, 0, 0, 0
def check_port(target, port, timeout=5):
"""
Проверить порт (port) на хосте (target)
:param target: хост
:param port: порт
:param timeout: таймаут
:return: True - порт открыт, False - нет хоста, порт закрыт, firewall
"""
try:
targetIP = socket.gethostbyname(target)
except socket.gaierror:
return False
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
try:
s.connect((targetIP, port))
s.close()
except (socket.error, socket.timeout):
return False
return True

Write Preview
Loading…
Cancel
Save