From 3834b7e1686cc9f3fd4af0a36d87826e7a3861d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=81?= Date: Fri, 16 Oct 2020 15:38:34 +0300 Subject: [PATCH] Template executor checks if the template is empty now and checks original file for changes using MD5. Format 'raw' is added. fixed #10 --- calculate/templates/format/base_format.py | 3 + calculate/templates/format/raw_format.py | 52 +++++++ calculate/templates/format/regex_format.py | 2 - calculate/templates/template_engine.py | 2 + calculate/templates/template_processor.py | 125 +++++++++------- pytest.ini | 1 + tests/templates/format/test_raw.py | 137 ++++++++++++++++++ tests/templates/test_directory_processor.py | 11 ++ .../etc.backup/img.gif | Bin 0 -> 3120 bytes .../templates_42/install/.calculate_directory | 1 + .../templates_42/install/template | 2 + 11 files changed, 284 insertions(+), 52 deletions(-) create mode 100644 calculate/templates/format/raw_format.py create mode 100644 tests/templates/format/test_raw.py create mode 100644 tests/templates/testfiles/test_dir_processor_root/etc.backup/img.gif create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_42/install/.calculate_directory create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_42/install/template diff --git a/calculate/templates/format/base_format.py b/calculate/templates/format/base_format.py index 1087dc0..c57086e 100644 --- a/calculate/templates/format/base_format.py +++ b/calculate/templates/format/base_format.py @@ -378,3 +378,6 @@ class Format(): self.CALCULATE_VERSION) document_text = re.sub(header_pattern, '', input_text) return header, document_text + + def __bool__(self): + return bool(self._document_dictionary) diff --git a/calculate/templates/format/raw_format.py b/calculate/templates/format/raw_format.py new file mode 100644 index 0000000..375669b --- /dev/null +++ b/calculate/templates/format/raw_format.py @@ -0,0 +1,52 @@ +# vim: fileencoding=utf-8 +# +from .base_format import Format +from ..template_engine import ParametersContainer + + +class RawFormat(Format): + '''Класс формата raw.''' + FORMAT = 'raw' + EXECUTABLE = False + FORMAT_PARAMETERS = {'comment'} + + def __init__(self, document_text: str, + template_path, + ignore_comments=False, + join_before=False, + add_header=False, + already_changed=False, + parameters=ParametersContainer()): + self.comment_symbol = parameters.comment or "#" + self._before = join_before + + if add_header and not ignore_comments: + self.header, self._document_text =\ + self._get_header_and_document_text( + document_text, + template_path, + already_changed=already_changed) + else: + self.header = '' + self._document_text = document_text + + @property + def document_text(self): + return '{}{}'.format(self.header, self._document_text) + + def join_template(self, template): + if self._before: + self._document_text = '{0}{1}{2}'.format( + template._document_text, + "" if template._document_text. + endswith("\n") else "\n", + self._document_text) + else: + self._document_text = '{0}{1}{2}'.format( + self._document_text, + "" if template._document_text. + endswith("\n") else "\n", + template._document_text) + + def __bool__(self): + return bool(self.document_text) diff --git a/calculate/templates/format/regex_format.py b/calculate/templates/format/regex_format.py index 3ab1d10..d1d1434 100644 --- a/calculate/templates/format/regex_format.py +++ b/calculate/templates/format/regex_format.py @@ -27,8 +27,6 @@ class RegexFormat(Format): processing_methods = OrderedDict() super().__init__(processing_methods) - self.changed_files = dict() - self._multiline_flag = parameters.multiline self._dotall_flag = parameters.dotall self._parsed_patch = None diff --git a/calculate/templates/template_engine.py b/calculate/templates/template_engine.py index 9ce10e9..b6f8b09 100644 --- a/calculate/templates/template_engine.py +++ b/calculate/templates/template_engine.py @@ -421,6 +421,8 @@ class ParametersProcessor: real_path = parameter_value # Ставим True, чтобы потом проверить этот параметр в postparse + print(f'source value {parameter_value}') + print(f'source <- {real_path}') if not os.path.exists(real_path): return True diff --git a/calculate/templates/template_processor.py b/calculate/templates/template_processor.py index 902f1b6..818d51b 100644 --- a/calculate/templates/template_processor.py +++ b/calculate/templates/template_processor.py @@ -949,35 +949,53 @@ class TemplateExecutor: chown = self.file_default_parameters.get('chown', False) if template_format.EXECUTABLE or template_object.md5_matching: - # Действия при совпадении md5 из CONTENTS и md5 целевого файла. - # А также если шаблон просто исполнительный. - output_paths = [output_path] + parsed_template = template_format(template_object.template_text, + template_object.template_path, + ignore_comments=True) - # Если целевой файл защищен, а шаблон не userspace. - if template_object.protected and not template_object.is_userspace: - # Тогда также обновляем архив. - output_paths.append(template_object.archive_path) + if (not parsed_template and not template_object.parameters.source + and template_object.format_class.EXECUTABLE): + if template_object.target_type is DIR: + self._remove_directory(template_object.target_path) + shutil.copytree(input_path, + template_object.target_path) + elif template_object.target_type is FILE: + self._remove_file(template_object.target_path) + shutil.copy(input_path, + template_object.template_path) + if chown: + self._chown_file(template_object.target_path, chown) - if template_object.target_type is not None and not replace: - # Если целевой файл есть и нет параметра replace -- используем - # текст целевого файла. - if (not input_path.startswith(self.cl_config_archive_path) or - os.path.exists(input_path)): - # Если входной файл просто не из архива, или из архива и - # при этом существует -- используем его - with open(input_path, 'r') as input_file: - input_text = input_file.read() + if chmod: + self._chmod_file(template_object.target_path, chmod) + + elif not template_object.format_class.EXECUTABLE: + # Действия при совпадении md5 из CONTENTS и md5 целевого файла. + # А также если шаблон просто исполнительный. + output_paths = [output_path] + + # Если целевой файл защищен, а шаблон не userspace. + if (template_object.protected + and not template_object.is_userspace): + # Тогда также обновляем архив. + output_paths.append(template_object.archive_path) + + if template_object.target_type is not None and not replace: + # Если целевой файл есть и нет параметра replace -- + # используем текст целевого файла. + if (not input_path.startswith(self.cl_config_archive_path) + or os.path.exists(input_path)): + # Если входной файл просто не из архива, или из архива + # и при этом существует -- используем его + with open(input_path, 'r') as input_file: + input_text = input_file.read() + else: + # В противном случае используем пустой файл. + # TODO Подумать. + input_text = '' else: - # В противном случае используем пустой файл. TODO Подумать. input_text = '' - else: - input_text = '' - - parsed_template = template_format(template_object.template_text, - template_object.template_path, - ignore_comments=True) - if not template_object.format_class.EXECUTABLE: # Если шаблон не исполнительный разбираем входной текст. parsed_input = template_format( input_text, @@ -988,11 +1006,14 @@ class TemplateExecutor: in self.processed_targets), parameters=template_object.parameters) parsed_input.join_template(parsed_template) + # Результат наложения шаблона. output_text = parsed_input.document_text + # Удаляем форматный объект входного файла. del(parsed_input) output_text_md5 = hashlib.md5(output_text.encode()).hexdigest() + input_text_md5 = hashlib.md5(input_text.encode()).hexdigest() for save_path in output_paths: if not os.path.exists(os.path.dirname(save_path)): @@ -1009,33 +1030,10 @@ class TemplateExecutor: if chmod: self._chmod_file(save_path, chmod) - - if self.dbpkg: - # Убираем все ._cfg файлы. - if template_object.cfg_list: - for cfg_file_path in template_object.cfg_list: - self._remove_file(cfg_file_path) - - # Убираем целевой файл из CL. - self.calculate_config_file.remove_file( - template_object.target_path) - - if template_object.target_type is None: - self.changed_files[template_object.target_path] = 'N' - else: - self.changed_files[template_object.target_path] = 'M' - - # Обновляем CONTENTS. - if template_object.protected: - if template_object.parameters.unbound: - template_object.remove_from_contents() - else: - template_object.add_to_contents( - file_md5=output_text_md5) - else: + elif template_object.format_class.EXECUTABLE: changed_files = parsed_template.execute_format( - input_text=input_text, - target_path=template_object.target_path) + template_object.target_path, + chroot_path=self.chroot_path) # Удаляем форматный объект входного файла. del(parsed_template) # Если исполняемый формат выдал список измененных файлов для @@ -1062,6 +1060,33 @@ class TemplateExecutor: template_object.target_package and template_object.format_class.FORMAT != 'contents'): template_object.update_contents_from_list(changed_files) + return + + if input_text_md5 != output_text_md5: + if template_object.target_type is None: + self.changed_files[ + template_object.target_path] = 'N' + else: + self.changed_files[ + template_object.target_path] = 'M' + + if self.dbpkg: + # Убираем все ._cfg файлы. + if template_object.cfg_list: + for cfg_file_path in template_object.cfg_list: + self._remove_file(cfg_file_path) + + # Убираем целевой файл из CL. + self.calculate_config_file.remove_file( + template_object.target_path) + + # Обновляем CONTENTS. + if template_object.protected: + if template_object.parameters.unbound: + template_object.remove_from_contents() + else: + template_object.add_to_contents( + file_md5=output_text_md5) else: if template_object.target_type is not None and not replace: if (not input_path.startswith(self.cl_config_archive_path) or diff --git a/pytest.ini b/pytest.ini index 88cf477..90cd471 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,6 +13,7 @@ markers = kernel: marker for running test for kernel format. ldap: marker for running test for ldap format. openrc: marker for running test for openrc format. + raw: marker for running test fot raw format. regex: marker for running test fot regex format. postfix: marker for running test for postfix format. procmail: marker for running test for procmail format. diff --git a/tests/templates/format/test_raw.py b/tests/templates/format/test_raw.py new file mode 100644 index 0000000..0480c9c --- /dev/null +++ b/tests/templates/format/test_raw.py @@ -0,0 +1,137 @@ +import pytest +from collections import OrderedDict +from calculate.templates.format.raw_format import RawFormat + + +@pytest.mark.raw +class TestParsingMethods: + def test_first(self): + original = '''parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + template = '''parameter-4 = d; +parameter-5 = e;''' + + output = '''parameter-1 = a; +parameter-2 = b; +parameter-3 = c; +parameter-4 = d; +parameter-5 = e;''' + + raw_document_1 = RawFormat(original, 'path/to/template') + raw_document_2 = RawFormat(template, 'path/to/template') + raw_document_1.join_template(raw_document_2) + + assert raw_document_1.document_text == output + + def test_second(self): + document_1 = '''parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + document_2 = '''parameter-4 = d; +parameter-5 = e;''' + + output = '''parameter-4 = d; +parameter-5 = e; +parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + raw_document_1 = RawFormat(document_1, 'path/to/template', + join_before=True) + raw_document_2 = RawFormat(document_2, 'path/to/template') + raw_document_1.join_template(raw_document_2) + + assert raw_document_1.document_text == output + + def test_first_with_comment(self): + document_1 = '''parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + document_2 = '''parameter-4 = d; +parameter-5 = e;''' + + output = '''#------------------------------------------------------------------------------- +# Modified by Calculate Utilities 4.0 +# Processed template files: +# path/to/template +#------------------------------------------------------------------------------- +parameter-1 = a; +parameter-2 = b; +parameter-3 = c; +parameter-4 = d; +parameter-5 = e;''' + + raw_document_1 = RawFormat(document_1, 'path/to/template', + add_header=True) + raw_document_2 = RawFormat(document_2, 'path/to/template') + raw_document_1.join_template(raw_document_2) + + assert raw_document_1.document_text == output + + def test_second_with_comment(self): + document_1 = '''parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + document_2 = '''parameter-4 = d; +parameter-5 = e;''' + + output = '''#------------------------------------------------------------------------------- +# Modified by Calculate Utilities 4.0 +# Processed template files: +# path/to/template +#------------------------------------------------------------------------------- +parameter-4 = d; +parameter-5 = e; +parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + raw_document_1 = RawFormat(document_1, 'path/to/template', + join_before=True, add_header=True) + raw_document_2 = RawFormat(document_2, 'path/to/template') + raw_document_1.join_template(raw_document_2) + + assert raw_document_1.document_text == output + + def test_third_with_comment(self): + original_text = '''parameter-1 = a; +parameter-2 = b; +parameter-3 = c;''' + + template_text_1 = '''parameter-4 = d; +parameter-5 = e;''' + + template_text_2 = '''parameter-6 = f;''' + + output = '''#------------------------------------------------------------------------------- +# Modified by Calculate Utilities 4.0 +# Processed template files: +# path/to/template_1 +# path/to/template_2 +#------------------------------------------------------------------------------- +parameter-1 = a; +parameter-2 = b; +parameter-3 = c; +parameter-4 = d; +parameter-5 = e; +parameter-6 = f;''' + + original_1 = RawFormat(original_text, 'path/to/template_1', + add_header=True, + already_changed=False) + template_1 = RawFormat(template_text_1, 'path/to/template_1') + template_2 = RawFormat(template_text_2, 'path/to/template_2') + + original_1.join_template(template_1) + original_2 = RawFormat(original_1.document_text, 'path/to/template_2', + add_header=True, + already_changed=True) + + original_2.join_template(template_2) + + assert original_2.document_text == output diff --git a/tests/templates/test_directory_processor.py b/tests/templates/test_directory_processor.py index 5101ca9..cdf2a0b 100644 --- a/tests/templates/test_directory_processor.py +++ b/tests/templates/test_directory_processor.py @@ -1379,6 +1379,17 @@ class TestDirectoryProcessor: assert '/etc/dir_59' in test_package_0 assert '/etc/dir_59/file_0' in test_package_0 + def test_copy_file_using_source(self): + datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH, + 'templates_42') + directory_processor = DirectoryProcessor( + 'install', + datavars_module=datavars, + package='test-category/test-package' + ) + directory_processor.process_template_directories() + assert os.path.exists(join_paths(CHROOT_PATH, '/etc/copy.gif')) + def test_view_tree(self): list_path = join_paths(CHROOT_PATH, '/etc') show_tree(list_path) diff --git a/tests/templates/testfiles/test_dir_processor_root/etc.backup/img.gif b/tests/templates/testfiles/test_dir_processor_root/etc.backup/img.gif new file mode 100644 index 0000000000000000000000000000000000000000..c255aaa895f0737629aa628f6262792ee41a2d9a GIT binary patch literal 3120 zcmV-04A1jYNk&E}3;+OEMM6+kP&iB+3;+NxjX)Ftx8ooH+eCV1w(y>jHHQK+5=5maKswN~uqcBf;QtHCsR$Kdc=S7`+A>`_4|6y$ItEh*6y5L#x1w|T{2K8mkg`(&!q7il2uV( zQr=Ycn&rQS8h>V?f7HJSBTGin3p$ja;7`z#dMQR)mEHrOl7GC8Ft5&Z>!RN%^mU+B zQ!Vdmv58rMbH#wA@gVB6u2-0fdmCw(7j>6$(`LPJUV^(j z-NDj!akJlEY)3WMowpv&8N6-V_9oS~ZP^JDT5J&%5J7*>*Z zKRo(q|9?vMeGjON&+8V^e~6N81#Qmbg(pS42tmyB3)tgYwoJz-s1^6v8^E%5!#V~| zN@TiJPO$a1y*kPzBBC-?OjNCAb*+PP%d7xI1emaDHossaGWu5IO0g+|?ly{4(ubZpd!EFz*>z2vm)aNZU(x-`svZn~eW>HVm!T^44uN z(Li^gh}Lbj>G>CnFK_SY^RMPVOvcYf@5R@~d!2R*(g;BR^>a)x{r;=ZZ|VPZOP~Mn z``>HhryZhz2*6t}-+KA)k6yv#$M5L*Yxm#3YiqLd_j}#bp0b3!_~iMEm-L!mzIq9K z0({5T!V8=Gao2xO7GKo| zMWIh`i1-cTtv8D!`(|;hwfDOH;1<{SLakzmi1I-T9<6CSu=Nmc$)IsspO>fXKw2Xe zB2tNnLc|)v&cQ}kL2C`YpXW=2MCkf$CPPG)g7TW|Fck?c-H*b&@j)t|<>yOF!CJf< zxP!g`6+^^D0ZE)MdS5y1wX1fGd%4TAr)kMf55gMK%Zbqj05KpU;%`V<>9(nfhFiB$ zbDDN}?}i?Z?VR>RS4k+*h=?MD zFc2_Zs!FHb>Gt|W)a^8@W~E#*PoB^erVbGa-etV=C42zL=Zp7er7!!|gL+Lb9h=9A zr}Ki;?nq);6y=OB1VG&Qa77&*pkDpZn z0njyzg`@ob6Zq|;-DtK1A8_tR{@(zA3;3pm@$oZV0nk5f)XJrDqb0mVv`-58!>szK zB|xL)Hk#u0AQuWFB8d`2M0`L*1c1b>(D6`Yx0t zc@qKvBqTq}wX#Ms9Acy&9cPd7lnU*~w=1X0pjF6a_aCj$f$h{!1s|l;@P_9ax7ZSC zq!W=C2;&oravAF$>_kF|iY0$h2x&RDPfj+oZzLY4wMNZ$M28jPJfTW-Nf3sJOB(ry z?Q}x7s>$G?n^ui5P=Goqz>!zUcRkD|*8sr4Ma1X2pusQlGON^JPs%qdpAqDVNzQS$3*PK5zTlcdX9 zT9+V^VsIKbaRvf^Ftf{K)_F(YBNdLsIF}=AVKzX1u)&6t`RpApd6jwGA^>Cv$T=>- z;Jci6iHf`g6Z{2=E|I0dbg94$%OyJ?a7yJ$b)fWpKlEGn{_d4+(+)|I?(MvS5Sw2E zS3?%{4ppDDn~`;d={DbW!z<6bh8)AqaU!`7)7CW$Qm-7Jl&P-_4r2hi9u0LPn$Wkl zcY;tw0{}MS^GM|xpz&XfTmdM#WQsTp0OaVF>r^CICIHD{+AJQInxfl2 zDeqYKJpjRl0^wl9;)z(~hyj3UYYbGgS=!ArSSABkO=B=K9D)>b4W>yb00!xzpA?G6 zHGh2rjBYvaqi)2wJQx5FuHrGAE#_X8q(_eLf@|kt@;Uz^@tMb7PmbB*=|Ryn=X=5ZM#%XvJ~O33Jf`P66X1i*(w_DE(L9>*sXLxJ)VHPwzsSW9ku zq{!IaUWK{1!O=nt;{~SGf?~JWvR;{gc!?7(ReUJgtq{(+OlD)%5L>1v5+gFkm<;<6H zz9PWAc#*|uHAlWi17$O+?|Kj*g(4f0T~#zDV|)!jdS@^Eo3b((i%qQl+U;dlB(amq z5YYi-#A8tsH^7NxSOTX@(i@46RS~{%XAkhhkyRV?)!5da4;@4$Z0h2Bla8%z8S}vI z-g76f4hi$>0sgPY>L`^+a(ACgGIn=po3|anhy<{Z3jcFO0%Xh|X7+^V#+Yp# zNbLzQ^kpoTi{4pWTpxwLMDggGe)xA7%#!a%_q3! zo}M>%pIk5C`8961M86y!u%Ymbp9HLts{?McH5JuN7nkjY=F^E_4}dr$r`)X@PB{_xoUqY^zpdxu4k&=%zWwh@UzZnypGKU+ zE2K~9yIrosf%Xsx8Lp-iOs}pXs2k6(#PcQHxtKG^h0pzQ@cd4~ov#9)eN8vtzs-&4 zWAM(Es8zaAkc{t)Nq5dcoO^TB&+q0Jg7i=D?Qie15MlV?Z^SUUdJLpV`1%P*zU9N& zatyybUtE6j>3`pT@`cw5H$FeT%Yvj$E*=5ABOS~6h{>3ZB(q_u{rI~-;n#N=@b*t% z`fc5P7h%iT=zfCpId2S@N-r>#%6rGx^Nt|me#TX}m~kr{S3H~C59PfXPtI4^LwxZ4 K*WYk8*31BKJ`olG literal 0 HcmV?d00001 diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_42/install/.calculate_directory b/tests/templates/testfiles/test_dir_processor_root/templates_42/install/.calculate_directory new file mode 100644 index 0000000..3711826 --- /dev/null +++ b/tests/templates/testfiles/test_dir_processor_root/templates_42/install/.calculate_directory @@ -0,0 +1 @@ +{% calculate action = "install", append = "skip", path = "/etc" %} diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_42/install/template b/tests/templates/testfiles/test_dir_processor_root/templates_42/install/template new file mode 100644 index 0000000..08ae45b --- /dev/null +++ b/tests/templates/testfiles/test_dir_processor_root/templates_42/install/template @@ -0,0 +1,2 @@ +{% calculate append = "join", format = "raw", source = "/etc/img.gif", +name = "copy.gif", package = "test-category/test-package" -%}