From 6a85c6b9399996b1db587327349de25c90699687 Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Wed, 17 Dec 2014 17:25:00 +0300 Subject: [PATCH 1/6] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/cl_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pym/cl_utils.py b/pym/cl_utils.py index cd2acc5..5c6a199 100644 --- a/pym/cl_utils.py +++ b/pym/cl_utils.py @@ -124,7 +124,13 @@ class typeFile: def getMType(self, filename): """Информация о типе файла""" - ret = self.magicObject.file(filename) + try: + ret = self.magicObject.file(filename) + except UnicodeDecodeError: + try: + ret = self.magicObject.file(filename.decode('utf-8')) + except UnicodeDecodeError: + return None # fix for kernel 3.7.7 (bad work samba with big files) if ret is None and self.magicObject.errno() == 5: r,w = os.pipe() From 21a3be386588d9c35492f3c91a01757a2f4d1d1c Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Thu, 25 Dec 2014 15:47:55 +0300 Subject: [PATCH 2/6] Add cleaning kernel and initrd --- pym/cl_utils.py | 33 ++++++++++++++ pym/cl_vars_share.py | 106 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 135 insertions(+), 4 deletions(-) diff --git a/pym/cl_utils.py b/pym/cl_utils.py index 5c6a199..da5ef2e 100644 --- a/pym/cl_utils.py +++ b/pym/cl_utils.py @@ -1479,3 +1479,36 @@ def countFiles(dirpath,onefilesystem=True): for dirname in mountDirs: dirnames.remove(dirname) return num + +class InitrdFile(object): + re_kver_path = re.compile("/modules/([^/]+)/kernel") + def __init__(self, _file): + self._file = _file + + def get_kernel_version(self): + for fn in self.get_names(): + if "/modules/" in fn and "/kernel" in fn: + m = self.re_kver_path.search(fn) + if m: + return m.group(1) + else: + break + return "" + + def get_names(self): + if not path.exists(self._file): + # raise IOError + open(self._file) + gz = Popen(['/bin/gzip', "-dc", self._file], stdout=PIPE, stderr=PIPE, + close_fds=True) + cpio = Popen(["/bin/cpio","-tf"], stdout=PIPE, stdin=gz.stdout, + stderr=PIPE, close_fds=True) + try: + for fn in cpio.stdout.xreadlines(): + yield fn.rstrip() + finally: + cpio.terminate() + gz.terminate() + + def __iter__(self): + return iter(self.get_names()) diff --git a/pym/cl_vars_share.py b/pym/cl_vars_share.py index 52a114e..ec4981a 100644 --- a/pym/cl_vars_share.py +++ b/pym/cl_vars_share.py @@ -16,9 +16,11 @@ import os import sys -from cl_utils import getpathenv, runOsCommand, typeFile,process +from cl_utils import (getpathenv, runOsCommand, typeFile,process, + listDirectory, InitrdFile) import re from os import path, R_OK + try: from cl_ldap import ldapUser except ImportError: @@ -276,10 +278,13 @@ class varsShare: def getFilesByType(self,pathname,descr): """Get files from "pathname" has "descr" in descriptions""" - filelist = map(lambda x:path.join(pathname,x),os.listdir(pathname)) + return list(self.get_files_by_type(pathname, descr)) + + def get_files_by_type(self, pathname, descr): ftype = typeFile(magic=0x4).getMType - filesWithType = map(lambda x:(x,ftype(x)), filelist) - return filter(lambda x:descr in x[1],filesWithType) + for x in listDirectory(pathname, fullPath=True): + if descr in ftype(x): + yield x def getInitrd(self,arch,shortname,chroot,kernel,suffix="",notsuffix=""): """Get initrd for kernel""" @@ -629,3 +634,96 @@ class varsShare: return res[:8] else: return "no_uid" + + def get_current_kernel_src(self, prefix='/'): + """ + Get current kernel source directory + """ + src_path = "usr/src" + current_linux_src = path.join(src_path,"linux") + symlink_kernel = path.join(prefix,current_linux_src) + if not path.exists(symlink_kernel) or not path.islink(symlink_kernel): + raise ValueError("Failed to determine current kernel version") + return path.join(src_path,os.readlink(symlink_kernel)) + + def get_config_version(self, configfile): + re_config = re.compile("Automatically generated file;.*\n" + ".*?Linux/\S+\s+(\S+)\s", re.M) + if path.exists(configfile): + with open(configfile) as f: + match = re_config.search(f.read(200)) + if match: + return match.group(1) + + def get_src_kernel_version(self, kernel_src): + """ + Get version of kernel from .config + """ + config_path = path.join(kernel_src, ".config") + makefile_path = path.join(kernel_src, "Makefile") + + # get version from config + version = self.get_config_version(config_path) + if version: + return version + + # get version from Makefile + re_makefile = re.compile("^VERSION = (\S+)\n" + "PATCHLEVEL = (\S+)\n" + "SUBLEVEL = (\S+)\n" + "EXTRAVERSION = (\S*)\n", re.M) + if path.exists(makefile_path): + with open(makefile_path) as f: + match = re_makefile.search(f.read(200)) + if match: + return "{0}.{1}.{2}{3}".format(*match.groups()) + return "" + + def list_initramfs(self, prefix='/', bootdir='boot'): + boot_dir = path.join(prefix, bootdir) + return self.get_files_by_type(boot_dir,"ASCII cpio archive") + + def filter_initramfs(self, iterable, version=None): + for fn in iterable: + if InitrdFile(fn).get_kernel_version() == version: + yield fn + + def list_kernel(self, prefix='/', bootdir='boot'): + boot_dir = path.join(prefix, bootdir) + return self.get_files_by_type(boot_dir, "boot executable bzImage") + + def filter_kernel(self, iterable, version=None): + ftype = typeFile(magic=0x4).getMType + re_kver = re.compile("bzImage, version (\S+)\s") + for fn in iterable: + m = re_kver.search(ftype(fn)) + if m.group(1) == version: + yield fn + + def list_config(self, prefix='/', bootdir='boot'): + boot_dir = path.join(prefix, bootdir) + return self.get_files_by_type(boot_dir, "Linux make config build file") + + def filter_config(self, iterable, version=None): + for fn in iterable: + if self.get_config_version(fn) == version: + yield fn + + def list_system_map(self, prefix='/', bootdir='boot'): + boot_dir = path.join(prefix, bootdir) + for fn in listDirectory(boot_dir): + if fn.startswith("System.map"): + yield path.join(boot_dir, fn) + + def filter_system_map(self, iterable, version=None): + re_kver = re.compile("System.map-(\S+)$") + for fn in iterable: + m = re_kver.search(fn) + if m and m.group(1) == version: + yield fn + + def max_default(self, iterable, key=lambda x:x, default=None): + try: + return max(iterable, key=key) + except ValueError: + return default From fc4dc1ee5aac89800899c9ba3486761250481190 Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Thu, 26 Mar 2015 16:50:41 +0300 Subject: [PATCH 3/6] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=B7=D0=B0=D0=BF=D0=B8=D1=81=D1=8C=20?= =?UTF-8?q?=D0=B2=20ini=20=D1=84=D0=B0=D0=B9=D0=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/cl_template.py | 96 ++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/pym/cl_template.py b/pym/cl_template.py index 99aabf6..d9bfdee 100644 --- a/pym/cl_template.py +++ b/pym/cl_template.py @@ -5380,6 +5380,9 @@ class templateClt(scanDirectoryClt, template): return False return (self.createdDirs, self.filesApply) +class IniError(Exception): + pass + class iniParser(_error, templateFormat): """Класс для работы с ini файлами @@ -5408,6 +5411,16 @@ class iniParser(_error, templateFormat): def openRWIniFile(self): if not os.access(self.iniFile, os.R_OK): + if not os.path.exists(self.iniFile): + try: + # Создание файла + self.FD = open(self.iniFile, "w+") + fcntl.flock(self.FD.fileno(), fcntl.LOCK_EX) + os.chmod(self.iniFile, self.mode) + except: + error = _("Unable to create the file") + ": " + self.iniFile + self.setError(error) + raise IniError(error) return "" try: self.FD = open(self.iniFile, "r+") @@ -5423,16 +5436,6 @@ class iniParser(_error, templateFormat): self.setError(_("Unable to write to the file") + ": " + self.iniFile) return False - if not os.path.exists(self.iniFile): - try: - # Создание файла - self.FD = open(self.iniFile, "w+") - fcntl.flock(self.FD.fileno(), fcntl.LOCK_EX) - os.chmod(self.iniFile, self.mode) - except: - self.setError(_("Unable to create the file") + - ": " + self.iniFile) - return False if not self.FD: self.setError(_("Unable to write to the file") + ": " + self.iniFile) @@ -5440,8 +5443,6 @@ class iniParser(_error, templateFormat): self.FD.truncate(0) self.FD.seek(0) self.FD.write(txtConfig) - self.FD.close() - self.FD = None return True def setVar(self, strHeader, dictVar): @@ -5451,37 +5452,50 @@ class iniParser(_error, templateFormat): strHeader - имя области dictVar - словарь переменных """ - textIni = self.openRWIniFile() - nameFomat = self.checkIniFile(textIni) - if not nameFomat: - return False - if type(strHeader) in (tuple, list): - # формат plasma - classObj = self.getClassObj("plasma") - else: - if nameFomat == "plasma": - self.setError(_("Trying to write a variable of 'samba' format" - " to file %s (format - 'plasma')") - %self.iniFile) + try: + textIni = self.openRWIniFile() + nameFomat = self.checkIniFile(textIni) + if not nameFomat: return False - # формат samba - classObj = self.getClassObj("samba") - # создаем объект - # и записываем в него содержимое ini-файла - objIni = classObj(textIni) - # создаем текст из строки заголовка и - # словаря переменных области - txtConfig = objIni.createTxtConfig(strHeader, dictVar) - # создаем объект и записываем в него текст - objIniAdd = classObj(txtConfig) - # объединяем объекты для получения результирующего текста - objIni.join(objIniAdd) - # получаем текст - txtConfig = objIni.getConfig().encode("UTF-8") - # записываем его в ini файл - if not self.writeIniFile(txtConfig): + if type(strHeader) in (tuple, list): + # формат plasma + classObj = self.getClassObj("plasma") + else: + if nameFomat == "plasma": + self.setError(_("Trying to write a variable of 'samba' format" + " to file %s (format - 'plasma')") + %self.iniFile) + return False + # формат samba + classObj = self.getClassObj("samba") + # создаем объект + # и записываем в него содержимое ini-файла + objIni = classObj(textIni) + # создаем текст из строки заголовка и + # словаря переменных области + txtConfig = objIni.createTxtConfig(strHeader, dictVar) + # создаем объект и записываем в него текст + objIniAdd = classObj(txtConfig) + # объединяем объекты для получения результирующего текста + objIni.join(objIniAdd) + # получаем текст + txtConfig = objIni.getConfig().encode("UTF-8") + # записываем его в ini файл + if not self.writeIniFile(txtConfig): + return False + return True + except IniError: return False - return True + finally: + self._closeFD() + + def _closeFD(self): + if self.FD: + try: + self.FD.close() + except: + pass + self.FD = None def isEmptyFile(self, textIni): """Если файл пустой или содержит только комментарии - False From 3d74aa27f715a671ade16598254c4fbc9addc419 Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Thu, 26 Mar 2015 16:51:13 +0300 Subject: [PATCH 4/6] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D0=BF=D1=80=D0=B5=D0=BD=D0=BE=D1=81=20xattr=20=D0=B0?= =?UTF-8?q?=D1=82=D1=82=D1=80=D0=B8=D0=B1=D1=83=D1=82=D0=BE=D0=B2=20=D0=BF?= =?UTF-8?q?=D1=80=D0=B8=20=D0=BA=D0=BE=D0=BF=D0=B8=D1=80=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=B8=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B7=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/cl_distr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/cl_distr.py b/pym/cl_distr.py index 4a7ebf2..0470779 100644 --- a/pym/cl_distr.py +++ b/pym/cl_distr.py @@ -507,7 +507,7 @@ class Distributive(object, SignalInterrupt): if not rsyncCmd: raise DistributiveError(_("Failed to find '%s' command")%"rsync") try: - rsyncProcess = process(rsyncCmd, "-a","-H", "-x", + rsyncProcess = process(rsyncCmd, "-a","-H", "-x","-XX", "%s/"%fromdir,"%s/"%todir,stderr=STDOUT) self.addInterruptProcess(rsyncProcess) res = rsyncProcess.success() From 90902838cd62f062bf56e49a54c0c6edc2fc45ed Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Thu, 26 Mar 2015 16:51:37 +0300 Subject: [PATCH 5/6] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BF=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6=D0=BA?= =?UTF-8?q?=D0=B0=20lz4=20=D1=81=D0=B6=D0=B0=D1=82=D0=B8=D1=8F=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20initramfs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/cl_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pym/cl_utils.py b/pym/cl_utils.py index da5ef2e..2e1a607 100644 --- a/pym/cl_utils.py +++ b/pym/cl_utils.py @@ -1499,7 +1499,15 @@ class InitrdFile(object): if not path.exists(self._file): # raise IOError open(self._file) - gz = Popen(['/bin/gzip', "-dc", self._file], stdout=PIPE, stderr=PIPE, + ftype = typeFile(magic=0x4).getMType + rdtype = ftype(self._file) + if "LZ4" in rdtype: + arch_cmd = '/usr/bin/lz4' + elif "XZ" in rdtype: + arch_cmd = '/usr/bin/xz' + else: + arch_cmd = '/bin/gzip' + gz = Popen([arch_cmd, "-dc", self._file], stdout=PIPE, stderr=PIPE, close_fds=True) cpio = Popen(["/bin/cpio","-tf"], stdout=PIPE, stdin=gz.stdout, stderr=PIPE, close_fds=True) From 1c1b5f8544ffa79eafc91d50515de1a8cd47121d Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Thu, 26 Mar 2015 17:07:17 +0300 Subject: [PATCH 6/6] =?UTF-8?q?=D0=9F=D1=80=D0=BE=D0=BF=D1=83=D1=81=D0=BA?= =?UTF-8?q?=20=D1=88=D0=B0=D0=B1=D0=BB=D0=BE=D0=BD=D0=BE=D0=B2=20=D1=81=20?= =?UTF-8?q?=D1=83=D0=BA=D0=B0=D0=B7=D0=B0=D0=BD=D0=BD=D1=8B=D0=BC=20env=20?= =?UTF-8?q?=D0=BF=D0=B0=D1=80=D0=B0=D0=BC=D0=B5=D1=82=D1=80=D0=BE=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/cl_template.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pym/cl_template.py b/pym/cl_template.py index d9bfdee..a822251 100644 --- a/pym/cl_template.py +++ b/pym/cl_template.py @@ -4233,6 +4233,9 @@ re.M|re.S) templateDirFile) return ("", False, []) + if " env=" in textTemplate: + return ("", False, []) + # Заменяем переменные на их значения textTemplate = self.applyVarsTemplate(textTemplate, templateDirFile) @@ -4878,6 +4881,8 @@ re.M|re.S) self.closeTemplFile() # Флаг копирования шаблона в конфигурационный файл flagCopyTemplate = True + if " env=" in self.textTemplate.partition('\n')[0]: + return False # Тип шаблона бинарный или текстовый templateFileType = self.getTemplateType() if templateFileType != "bin":