From 0ae8133cf2fa278448091dbbc1a8fc711090fd10 Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Mon, 23 Sep 2024 14:36:03 +0200 Subject: [PATCH 1/7] Virtualenv and pre-commit implementation --- hooks/post_gen_project.py | 42 ++++++++++++++++++- .../.pre-commit-config.yaml | 32 ++++++++++++++ .../requirements.txt | 3 +- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 {{cookiecutter.repository_folder_name}}/.pre-commit-config.yaml diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index bda63d9..0b2f623 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -1,4 +1,5 @@ import os +import platform import shutil import subprocess @@ -15,11 +16,39 @@ def modify_portal_properties(repo_url): with open(Path('component_config/licenseUrl.md'), 'w') as inp: inp.write(repo_url+"/blob/master/LICENSE.md") + +def check_virtualenv_module() -> None: + if subprocess.run(["python", "-m", "virtualenv", "--version"]).returncode != 0: + print('ERROR: virtualenv module is not installed! Installing..."') + subprocess.run(["pip", "install", "virtualenv"]) + check_virtualenv_module() + else: + print('Module virtualenv is already installed. Proceeding...') -platform = '{{ cookiecutter.template_variant }}' + +def identify_os() -> str: + return platform.system().lower() + + +def create_venv_and_install_libraries(os_name: str) -> None: + print('Creating virtual environment...') + subprocess.run(["python", "-m", "virtualenv", "venv"]) + print('Virtual environment created. Installing libraries...') + + if os_name.startswith('win'): + subprocess.run(["venv\\Scripts\\pip", "install", "-r", "requirements.txt"]) + + else: + subprocess.run(["venv/bin/pip", "install", "-r", "requirements.txt"]) + + print('Libraries installed. Proceeding...') + + +# remove redundant files and directories +template_variant = '{{ cookiecutter.template_variant }}' repo_url = '{{ cookiecutter.repository_url }}' -if platform == 'GitHub': +if template_variant == 'GitHub': modify_portal_properties(repo_url=repo_url) REMOVE_PATHS = [ @@ -58,3 +87,12 @@ def handle_error(err_out): '\n WARNING: No repository_url was set. To set the remote to your repository please use following command:\n ' 'git remote add ' 'origin PATH_TO_YOUR_REPOSITORY') + + +# virtualenv setup process +os_name = identify_os() +check_virtualenv_module() +create_venv_and_install_libraries(os_name) + +project_name = '{{ cookiecutter.repository_folder_name }}' +print(f"\nProject \"{project_name}\" initialized successfully!") diff --git a/{{cookiecutter.repository_folder_name}}/.pre-commit-config.yaml b/{{cookiecutter.repository_folder_name}}/.pre-commit-config.yaml new file mode 100644 index 0000000..e32a410 --- /dev/null +++ b/{{cookiecutter.repository_folder_name}}/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +repos: + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + args: + - --max-line-length=120 + exclude: + - .git + - __pycache__ + - tests + - example + - venv + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + +# F812: list comprehension redefines ... +# H101: Use TODO(NAME) +# H202: assertRaises Exception too broad +# H233: Python 3.x incompatible use of print operator +# H301: one import per line +# H306: imports not in alphabetical order (time, os) +# H401: docstring should not start with a space +# H403: multi line docstrings should end on a new line +# H404: multi line docstring should start without a leading new line +# H405: multi line docstring summary not separated with an empty line +# H501: Do not use self.__dict__ for string formatting diff --git a/{{cookiecutter.repository_folder_name}}/requirements.txt b/{{cookiecutter.repository_folder_name}}/requirements.txt index 637d2fc..e4f05da 100644 --- a/{{cookiecutter.repository_folder_name}}/requirements.txt +++ b/{{cookiecutter.repository_folder_name}}/requirements.txt @@ -3,4 +3,5 @@ keboola.utils keboola.http-client freezegun mock -pydantic \ No newline at end of file +pydantic +pre-commit \ No newline at end of file From dfc48eb2e6eca08bd088524d6349a154c2c824a8 Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Tue, 24 Sep 2024 13:33:06 +0200 Subject: [PATCH 2/7] MacOS optimalization --- hooks/post_gen_project.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 0b2f623..cb2b8ec 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -1,10 +1,11 @@ import os -import platform import shutil import subprocess +import sys from pathlib import Path +python_exec = sys.executable def modify_portal_properties(repo_url): with open(Path('component_config/sourceCodeUrl.md'), 'w') as inp: @@ -18,7 +19,7 @@ def modify_portal_properties(repo_url): def check_virtualenv_module() -> None: - if subprocess.run(["python", "-m", "virtualenv", "--version"]).returncode != 0: + if subprocess.run([python_exec, "-m", "virtualenv", "--version"]).returncode != 0: print('ERROR: virtualenv module is not installed! Installing..."') subprocess.run(["pip", "install", "virtualenv"]) check_virtualenv_module() @@ -26,29 +27,28 @@ def check_virtualenv_module() -> None: print('Module virtualenv is already installed. Proceeding...') -def identify_os() -> str: - return platform.system().lower() - - -def create_venv_and_install_libraries(os_name: str) -> None: +def create_venv_and_install_libraries() -> None: print('Creating virtual environment...') - subprocess.run(["python", "-m", "virtualenv", "venv"]) - print('Virtual environment created. Installing libraries...') - - if os_name.startswith('win'): - subprocess.run(["venv\\Scripts\\pip", "install", "-r", "requirements.txt"]) + subprocess.run([python_exec, "-m", "virtualenv", "venv"]) + print('Virtual environment created.') + if os.name.lower().startswith('nt'): + pip_exec = os.path.join('venv', 'Scripts', 'pip') else: - subprocess.run(["venv/bin/pip", "install", "-r", "requirements.txt"]) + venv_path = os.path.join(os.getcwd(), "venv") + pip_exec = os.path.join(venv_path, 'bin', 'pip') + print('Installing libraries...') + subprocess.run([pip_exec, "install", "-r", "requirements.txt"]) print('Libraries installed. Proceeding...') + # remove redundant files and directories -template_variant = '{{ cookiecutter.template_variant }}' +platform = '{{ cookiecutter.template_variant }}' repo_url = '{{ cookiecutter.repository_url }}' -if template_variant == 'GitHub': +if platform == 'GitHub': modify_portal_properties(repo_url=repo_url) REMOVE_PATHS = [ @@ -90,9 +90,8 @@ def handle_error(err_out): # virtualenv setup process -os_name = identify_os() check_virtualenv_module() -create_venv_and_install_libraries(os_name) +create_venv_and_install_libraries() project_name = '{{ cookiecutter.repository_folder_name }}' print(f"\nProject \"{project_name}\" initialized successfully!") From bbf426e9e9e3a6f83b86f4a7976a687a94a1823d Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Tue, 24 Sep 2024 14:21:25 +0200 Subject: [PATCH 3/7] MacOS - shell script --- hooks/post_gen_project.py | 8 +++----- hooks/venv_setup.sh | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 hooks/venv_setup.sh diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index cb2b8ec..13b96fc 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -34,16 +34,14 @@ def create_venv_and_install_libraries() -> None: if os.name.lower().startswith('nt'): pip_exec = os.path.join('venv', 'Scripts', 'pip') + print('Installing libraries...') + subprocess.run([pip_exec, "install", "-r", "requirements.txt"]) else: - venv_path = os.path.join(os.getcwd(), "venv") - pip_exec = os.path.join(venv_path, 'bin', 'pip') + subprocess.run(["bash", "venv_setup.sh"]) - print('Installing libraries...') - subprocess.run([pip_exec, "install", "-r", "requirements.txt"]) print('Libraries installed. Proceeding...') - # remove redundant files and directories platform = '{{ cookiecutter.template_variant }}' repo_url = '{{ cookiecutter.repository_url }}' diff --git a/hooks/venv_setup.sh b/hooks/venv_setup.sh new file mode 100644 index 0000000..cd0ced3 --- /dev/null +++ b/hooks/venv_setup.sh @@ -0,0 +1,5 @@ +# venv_setup.sh - activates virtual environment and installs dependencies + +source venv/bin/activate +pip install -r requirements.txt +deactivate \ No newline at end of file From 6ea8cc498cf2fa43c763d2e27682d647c243fbcd Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Tue, 24 Sep 2024 14:26:19 +0200 Subject: [PATCH 4/7] back to hardcoded 'python' command --- hooks/post_gen_project.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 13b96fc..4bb5660 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -5,7 +5,6 @@ from pathlib import Path -python_exec = sys.executable def modify_portal_properties(repo_url): with open(Path('component_config/sourceCodeUrl.md'), 'w') as inp: @@ -19,7 +18,7 @@ def modify_portal_properties(repo_url): def check_virtualenv_module() -> None: - if subprocess.run([python_exec, "-m", "virtualenv", "--version"]).returncode != 0: + if subprocess.run(["python", "-m", "virtualenv", "--version"]).returncode != 0: print('ERROR: virtualenv module is not installed! Installing..."') subprocess.run(["pip", "install", "virtualenv"]) check_virtualenv_module() @@ -29,7 +28,7 @@ def check_virtualenv_module() -> None: def create_venv_and_install_libraries() -> None: print('Creating virtual environment...') - subprocess.run([python_exec, "-m", "virtualenv", "venv"]) + subprocess.run(["python", "-m", "virtualenv", "venv"]) print('Virtual environment created.') if os.name.lower().startswith('nt'): From 5ec208cf6de25216d8d1ef06cf78d8a4d4365091 Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Tue, 24 Sep 2024 14:51:37 +0200 Subject: [PATCH 5/7] better logging of the gen process --- hooks/post_gen_project.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 4bb5660..6f65c8a 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -1,7 +1,6 @@ import os import shutil import subprocess -import sys from pathlib import Path @@ -19,26 +18,26 @@ def modify_portal_properties(repo_url): def check_virtualenv_module() -> None: if subprocess.run(["python", "-m", "virtualenv", "--version"]).returncode != 0: - print('ERROR: virtualenv module is not installed! Installing..."') - subprocess.run(["pip", "install", "virtualenv"]) + print('\n[COOKIECUTTER][ERROR]: Virtualenv module is not installed! Installing...') + subprocess.run(["python", "-m", "pip", "install", "virtualenv"]) check_virtualenv_module() else: - print('Module virtualenv is already installed. Proceeding...') + print('\n[COOKIECUTTER][INFO]: Module virtualenv is already installed. Proceeding...') def create_venv_and_install_libraries() -> None: - print('Creating virtual environment...') + print('\n[COOKIECUTTER][INFO]: Creating virtual environment...') subprocess.run(["python", "-m", "virtualenv", "venv"]) - print('Virtual environment created.') + print('\n[COOKIECUTTER][INFO]: Virtual environment created.') if os.name.lower().startswith('nt'): pip_exec = os.path.join('venv', 'Scripts', 'pip') - print('Installing libraries...') + print('\n[COOKIECUTTER][INFO]: Installing libraries...') subprocess.run([pip_exec, "install", "-r", "requirements.txt"]) else: subprocess.run(["bash", "venv_setup.sh"]) - print('Libraries installed. Proceeding...') + print('\n[COOKIECUTTER][INFO]: Libraries installed. Proceeding...') # remove redundant files and directories @@ -64,31 +63,34 @@ def create_venv_and_install_libraries() -> None: def handle_error(err_out): if err_out: - print(f"Command failed with error: {err_out}") + print(f"\n[COOKIECUTTER][ERROR]: Command failed with error: {err_out}") exit(1) # initialize GitHub repository -print("Initializing github repository") +print("\n[COOKIECUTTER][INFO]: Initializing github repository") subprocess.run(["git", "init"]) if repo_url: - print(f'\nSetting up remote to {repo_url}') + print(f'\n[COOKIECUTTER][INFO]: Setting up remote to {repo_url}') subprocess.run(["git", "remote", "add", "origin", repo_url]) -print("\nAdding first commit") +print("\n[COOKIECUTTER][INFO]: Adding first commit") subprocess.run(["git", "add", "."]) subprocess.run(["git", "commit", "-m", '"Initial commit"']) if not repo_url: print( - '\n WARNING: No repository_url was set. To set the remote to your repository please use following command:\n ' + '\n [COOKIECUTTER][WARNING]: No repository_url was set. To set the remote to your repository please use following command:\n ' 'git remote add ' 'origin PATH_TO_YOUR_REPOSITORY') # virtualenv setup process +print("\n[COOKIECUTTER][INFO]: Virtual environment setup process starting...") check_virtualenv_module() +print("\n[COOKIECUTTER][INFO]: Virtual environment module was checked. Proceeding...") create_venv_and_install_libraries() +print("\n[COOKIECUTTER][INFO]: Virtual environment setup process completed.") project_name = '{{ cookiecutter.repository_folder_name }}' -print(f"\nProject \"{project_name}\" initialized successfully!") +print(f"\n[COOKIECUTTER][INFO]: Project \"{project_name}\" initialized successfully!") From ba368a19605bf7255d0e3ded73b855686d833028 Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Wed, 25 Sep 2024 12:27:43 +0200 Subject: [PATCH 6/7] del of bash file, back to orig logic --- hooks/post_gen_project.py | 7 ++++--- hooks/venv_setup.sh | 5 ----- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 hooks/venv_setup.sh diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 6f65c8a..687fa10 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -32,11 +32,12 @@ def create_venv_and_install_libraries() -> None: if os.name.lower().startswith('nt'): pip_exec = os.path.join('venv', 'Scripts', 'pip') - print('\n[COOKIECUTTER][INFO]: Installing libraries...') - subprocess.run([pip_exec, "install", "-r", "requirements.txt"]) + else: - subprocess.run(["bash", "venv_setup.sh"]) + pip_exec = os.path.join('venv', 'bin', 'pip') + print('\n[COOKIECUTTER][INFO]: Installing libraries...') + subprocess.run([pip_exec, "install", "-r", "requirements.txt"]) print('\n[COOKIECUTTER][INFO]: Libraries installed. Proceeding...') diff --git a/hooks/venv_setup.sh b/hooks/venv_setup.sh deleted file mode 100644 index cd0ced3..0000000 --- a/hooks/venv_setup.sh +++ /dev/null @@ -1,5 +0,0 @@ -# venv_setup.sh - activates virtual environment and installs dependencies - -source venv/bin/activate -pip install -r requirements.txt -deactivate \ No newline at end of file From 13a7a3ee01741f9aaf47897d78d9215a1bfe8ba8 Mon Sep 17 00:00:00 2001 From: Albert Sedlacek Date: Thu, 7 Nov 2024 09:57:08 +0100 Subject: [PATCH 7/7] del pre-commit from requirements + check for pre-commit on device --- hooks/post_gen_project.py | 15 ++++++++++++++- .../requirements.txt | 3 +-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 687fa10..d7f957f 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -15,7 +15,18 @@ def modify_portal_properties(repo_url): with open(Path('component_config/licenseUrl.md'), 'w') as inp: inp.write(repo_url+"/blob/master/LICENSE.md") - + +def check_precommit_module() -> None: + if subprocess.run(["python", "-m", "pip", "show", "pre-commit"]).returncode != 0: + print('\n[COOKIECUTTER][ERROR]: Pre-commit module is not installed! Installing..') + subprocess.run(["python", "-m", "pip", "install", "pre-commit"]) + else: + print('\n[COOKIECUTTER][INFO]: Module pre-commit is already installed. Proceeding...') + + print('\n[COOKIECUTTER][INFO]: Installing pre-commit hooks...') + subprocess.run(["pre-commit", "install"]) + + def check_virtualenv_module() -> None: if subprocess.run(["python", "-m", "virtualenv", "--version"]).returncode != 0: print('\n[COOKIECUTTER][ERROR]: Virtualenv module is not installed! Installing...') @@ -87,6 +98,8 @@ def handle_error(err_out): # virtualenv setup process +print("\n[COOKIECUTTER][INFO]: Pre-commit module checking...") +check_precommit_module() print("\n[COOKIECUTTER][INFO]: Virtual environment setup process starting...") check_virtualenv_module() print("\n[COOKIECUTTER][INFO]: Virtual environment module was checked. Proceeding...") diff --git a/{{cookiecutter.repository_folder_name}}/requirements.txt b/{{cookiecutter.repository_folder_name}}/requirements.txt index e4f05da..637d2fc 100644 --- a/{{cookiecutter.repository_folder_name}}/requirements.txt +++ b/{{cookiecutter.repository_folder_name}}/requirements.txt @@ -3,5 +3,4 @@ keboola.utils keboola.http-client freezegun mock -pydantic -pre-commit \ No newline at end of file +pydantic \ No newline at end of file