# -*- coding: utf-8 -*- # Copyright 2015-2016 Mir Calculate. http://www.calculate-linux.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from contextlib import contextmanager import hashlib import json import os import sys from os import path from calculate.lib.datavars import SimpleDataVars, Variable from calculate.install.distr import Distributive from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate) import time from calculate.lib.configparser import ConfigParser from calculate.lib.utils.files import (readFile, listDirectory, writeFile) from calculate.lib.utils.mount import isMount from calculate.lib.utils.git import Git from .datavars import BuilderError from calculate.lib.utils.tools import Locker, LockError from .datavars import builder_data _ = lambda x: x setLocalTranslate('cl_builder3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) UTF8 = 'utf-8' class Build(object): """ Сборка """ class Status(object): Worked, Registered, Removed, Broken, Unregistered = range(5) class Parameter(object): HashId = "hash" Directory = "directory" PkgDir = "pkgdir" References = "references" types = {HashId: Variable.Types.String, Directory: Variable.Types.String, References: Variable.Types.List, PkgDir: Variable.Types.String, } attrs = {'pkgdir': PkgDir, 'references': References} class Branches(object): Stable = Git.Reference.Master Unstable = Git.Reference.Update def __init__(self, build_id, distributive, parent, restore=False): """ @type distributive:Distributive @type parent:BuildStorage """ self.distributive = distributive self.parent = parent self.id = build_id self.pkgdir = "" self.default_branch = Build.Branches.Stable self.references = [] self._hash = None if not restore: self.parent.reserve_id(self.id) self.status = Build.Status.Registered @property def build_hash(self): if self._hash is None: hash_info = self.parent.get_build_hash(self.id) if hash_info is None: self._hash = self.generate_hash_id() else: self._hash = hash_info return self._hash def set_overlays(self, overlays): self.references = [self.default_branch for x in overlays] @property def directory(self): return self.distributive.getDirectory() def generate_hash_id(self): assemble_name = (self.id + self.distributive.getType().decode(UTF8) + unicode(time.time())).encode(UTF8) return hashlib.sha1(assemble_name).hexdigest() def save(self): """ Сохранить сборку """ self.parent.save_build(self) self.status = Build.Status.Worked def remove(self): """ Удалить сборку """ if self.status in (Build.Status.Registered, Build.Status.Worked, Build.Status.Broken): self.parent.remove_build(self) self.status = Build.Status.Removed self.id = None self._hash = None def close_distributive(self): """ Закрыть дистрибутив привязанный к сборке """ self.distributive.release() for child in self.distributive.childs: child.release() self.distributive.close() def restore(self): """ Восстановить сборку """ self.distributive.childs = [] self.save() return True class BuildStorage(object): """ Хранилище в котором находится информация о собираемом дистрибутиве """ ini_file = "/var/lib/calculate/calculate-builder/builder.env" data_dir = path.join(builder_data, "mount") def __init__(self, data_dir=None, ini_file=None): if ini_file: self.ini_file = ini_file if data_dir: self.data_dir = data_dir if not path.exists(self.data_dir): os.makedirs(self.data_dir) self.locker = Locker(path.basename(self.ini_file)) def _get_ini(self): ini = ConfigParser() ini.read(self.ini_file, encoding=UTF8) return ini def _get_ini_with_lock(self): try: with self.locker: return self._get_ini() except LockError: raise BuilderError("Failed to lock builder.env") def reserve_id(self, build_id): """ Зарезервировать id """ with self.modify_ini() as ini: if build_id in ini: raise BuilderError(_("Build ID '%s' already in use")) else: ini.add_section(build_id) @contextmanager def modify_ini(self): try: with self.locker: ini = self._get_ini() yield ini if not ini.sections() and path.exists(self.ini_file): os.unlink(self.ini_file) else: with open(self.ini_file, 'w') as f: ini.write(f) except LockError: raise BuilderError("Failed to lock assemble.env") def save_build(self, build): """ Сохранить информация о сборке :param build: сборка :return: """ build_id = build.id d = {Build.Parameter.Directory: build.distributive.getDirectory(), Build.Parameter.HashId: build.build_hash} for k, v in Build.Parameter.attrs.items(): d[v] = SimpleDataVars.serialize(Build.Parameter.types[v], getattr(build, k)) with self.modify_ini() as ini: for k, v in d.items(): ini[build_id][k] = v mount_dump_file = self.get_dump_name(build.build_hash) with writeFile(mount_dump_file) as f: json.dump(build.distributive.serialize(), f) return build_id def get_dump_name(self, assemble_hash): return path.join(self.data_dir, assemble_hash) def clear_garbage(self): """ Удалить все записи о точках монтирования, которые отсутствуют в assemble.env :return: """ ini = self._get_ini_with_lock() hashes = [ini.get(x, Build.Parameter.HashId, fallback="").encode(UTF8) for x in ini] for fn in listDirectory(self.data_dir): if fn not in hashes: try: os.unlink(path.join(self.data_dir, fn)) except (OSError, IOError): pass def __iter__(self): ini = self._get_ini_with_lock() return iter(sorted([x.encode(UTF8) for x in ini if x != 'DEFAULT'])) def clear_unmounted(self): """ Удалить все записи о собираемых дистрибутивах, если они не подмонтированы :return: """ with self.modify_ini() as ini: dist_dirs = [(x.encode(UTF8), ini.get(x, Build.Parameter.Directory, fallback="").encode(UTF8)) for x in ini.keys()] for k, v in dist_dirs: if v is None or not isMount(v): ini.remove_section(k) def get_build_hash(self, build_id): return self._get_ini_with_lock().get(build_id, Build.Parameter.HashId, fallback=None) def get_build(self, build_id): ini = self._get_ini_with_lock() build_hash = ini.get(build_id, Build.Parameter.HashId, fallback=None) if build_hash: build_dir = ini.get(build_id, Build.Parameter.Directory) data = readFile(self.get_dump_name(build_hash)) if data: try: data = json.loads(data) except ValueError: raise BuilderError( _("Failed to get distributive information")) dist = Distributive.unserialize(data) build = Build(build_id, dist, self, restore=True) proc_build_dir = path.join(build_dir, 'proc') if isMount(build_dir) and isMount(proc_build_dir): build.status = build.Status.Worked else: build.status = build.Status.Broken for param_name, ini_name in Build.Parameter.attrs.items(): value = ini.get(build_id, ini_name, fallback="") value = SimpleDataVars.unserialize( Build.Parameter.types[ini_name], value) setattr(build, param_name, value) return build def remove_build(self, build): try: fn = path.join(self.data_dir, build.build_hash) if path.exists(fn): os.unlink(fn) except (OSError, IOError): pass with self.modify_ini() as ini: ini.remove_section(build.id)