Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
sqlite.db
venv*
.venv
logs/tdp.log
files
.ansible
.cache
.poetrycache
.uvcache
.vagrant*
ansible_collections/community*
tdp_vars
tests/__pycache__
.tmp
16 changes: 7 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,29 +54,29 @@ Ansible configuration has been preconfigured for the Vagrant setup in the `ansib

### TDP-dev python environment setup

Now to setup the python dependecis for TDP-dev which are marked in the poetry.lock file at the root of the project we are going to use a container.
Now to setup the python dependecis for TDP-dev which are marked in the uv.lock file at the root of the project we are going to use a container.

First build the image and run the container:

```sh
# Build command:
docker build -t tdp-dev dev

# make the .poetrycache folder
mkdir .poetrycache
# make the .uvcache folder
mkdir .uvcache

# Run command:
docker run --rm -it \
-v $PWD:/home/tdp/tdp-dev \
-v $PWD/.poetrycache:/home/tdp/.cache/pypoetry \
-v $PWD/.uvcache:/home/tdp/.cache/uv \
--network=host \
--env CONTAINER_UID=$(id -u) --env CONTAINER_GID=$(id -g) \
--env DISPLAY=$DISPLAY \
tdp-dev
```

- `-v $PWD:/home/tdp/tdp-dev` binds the working directory of your container to this repository in your host.
- `-v $PWD/.poetrycache:/home/tdp/.cache/pypoetry` binds the `.poetrycache` folder to the poetry cache in the container.
- `-v $PWD/.uvcache:/home/tdp/.cache/uv` binds the `.uvcache` folder to the uv cache in the container.
- With the `--network=host` option the container is connected to your host network which enables it to communicate with the VMs.
- The `--env CONTAINER_UID=$(id -u) --env CONTAINER_GID=$(id -g)` environment variables enable the container to have the same user as your host.
- The `--env DISPLAY=$DISPLAY` environment variable is for the tdp-lib command `tdp dag` to be able to display the dag in a new window on your host.
Expand All @@ -86,7 +86,7 @@ Inside the container create the `venv-dev` virtual environment which will contai
```sh
python -m venv venv-dev
source venv-dev/bin/activate
poetry install
uv sync --all-extras --active
```

TDP-lib is contained in the dependencies but not its development dependencies.
Expand Down Expand Up @@ -114,9 +114,7 @@ If you desire de develop TDP-lib with pytest, use the linter ruff, you will have
Inside the container create the `venv-lib` virtual environment and install the dependencies:

```sh
python -m venv venv-lib
source venv-lib/bin/activate
poetry install -C tdp-lib -E visualization -E mysql -E postgresql-binary
uv sync --all-extras --directory tdp-lib
```

Read the `tdp-lib` documentation for more information.
Expand Down
2 changes: 1 addition & 1 deletion ansible.cfg
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[defaults]
inventory=inventory/hosts,inventory/topologies,inventory/tdp_vars.yaml ; REQUIRED path to the directory containing the Ansible inventory.
collections_paths=. ; RECOMMENDED paths to the directories containing the Ansible collections. The first directory of this path is used by Ansible galaxy to install collections.
collections_path=. ; RECOMMENDED paths to the directories containing the Ansible collections. The first directory of this path is used by Ansible galaxy to install collections.
display_skipped_hosts=False ; RECOMMENDED to avoid displaying skipped Ansible tasks and cluttering the logs.
any_errors_fatal=True ; REQUIRED to stop Ansible execution as soon as an error occurs and prevent Ansible from continuing on the remaining hosts.

Expand Down
2 changes: 1 addition & 1 deletion ansible_collections/tosit/tdp
Submodule tdp updated 52 files
+4 −0 playbooks/hbase_kerberos_install.yml
+4 −0 playbooks/hdfs_kerberos_install.yml
+4 −0 playbooks/hive_kerberos_install.yml
+4 −0 playbooks/kerberos_spnego_install.yml
+4 −0 playbooks/spark3_kerberos_install.yml
+4 −0 playbooks/yarn_kerberos_install.yml
+4 −1 playbooks/zookeeper_kerberos_install.yml
+4 −55 plugins/inventory/tdp_vars.py
+4 −3 plugins/module_utils/kerberos.py
+4 −3 plugins/modules/krb_check_keytab.py
+10 −5 plugins/modules/krb_keytab.py
+0 −32 roles/hadoop/client/tasks/kerberos.yml
+1 −1 roles/hbase/master/tasks/hdfs_init.yml
+2 −2 roles/hbase/master/tasks/kerberos.yml
+2 −2 roles/hbase/regionserver/tasks/kerberos.yml
+2 −2 roles/hbase/rest/tasks/kerberos.yml
+1 −1 roles/hdfs/check/tasks/main.yml
+1 −1 roles/hdfs/common/tasks/hdfs_audit_init.yml
+2 −2 roles/hdfs/datanode/tasks/kerberos.yml
+2 −2 roles/hdfs/httpfs/tasks/kerberos.yml
+2 −2 roles/hdfs/journalnode/tasks/kerberos.yml
+2 −2 roles/hdfs/namenode/tasks/kerberos.yml
+2 −2 roles/hive/common/tasks/hdfs_init.yml
+2 −2 roles/hive/hiveserver2/tasks/kerberos.yml
+2 −2 roles/hive/metastore/tasks/kerberos.yml
+1 −1 roles/knox/gateway/tasks/hdfs_init.yml
+2 −2 roles/knox/gateway/tasks/kerberos.yml
+4 −4 roles/ranger/admin/tasks/kerberos.yml
+2 −2 roles/ranger/kms/tasks/kerberos.yml
+2 −2 roles/ranger/usersync/tasks/kerberos.yml
+1 −1 roles/spark/common/tasks/hdfs_init.yml
+2 −2 roles/spark/historyserver/tasks/kerberos.yml
+1 −1 roles/utils/hdfs_user_homes/tasks/main.yml
+0 −3 roles/utils/kerberos/tasks/create_headless_principal_keytab.yml
+1 −1 roles/utils/yarn_capacity_scheduler/tasks/main.yml
+2 −2 roles/yarn/apptimelineserver/tasks/kerberos.yml
+1 −1 roles/yarn/common/tasks/hdfs_init.yml
+2 −2 roles/yarn/jobhistoryserver/tasks/kerberos.yml
+2 −2 roles/yarn/nodemanager/tasks/kerberos.yml
+2 −2 roles/yarn/resourcemanager/tasks/kerberos.yml
+1 −1 roles/zookeeper/client/tasks/kerberos.yml
+1 −1 roles/zookeeper/common/templates/jaas.client.conf.j2
+2 −2 roles/zookeeper/server/tasks/kerberos.yml
+87 −19 tdp_vars_defaults/hadoop/hadoop.yml
+0 −4 tdp_vars_defaults/hdfs/hdfs_httpfs.yml
+9 −2 tdp_vars_defaults/kerberos/kerberos_spnego.yml
+1 −4 tdp_vars_defaults/knox/knox.yml
+4 −4 tdp_vars_defaults/ranger/ranger.yml
+1 −1 tdp_vars_defaults/spark3/spark3.yml
+18 −0 tdp_vars_defaults/tdp-cluster/tdp-cluster.yml
+1 −0 topology-light.ini
+1 −0 topology.ini
10 changes: 5 additions & 5 deletions dev/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM python:3.9-slim
FROM python:3.12-slim

# Install dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
Expand All @@ -12,11 +12,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
vim \
&& rm -rf /var/lib/apt/lists/*

# Location of poetry installation
ENV POETRY_HOME=/usr/local
# Location of uv installation
ENV UV_INSTALL_DIR=/usr/local/bin

# Install Poetry
RUN curl -sSL https://install.python-poetry.org | python -
# Install uv
RUN curl -LsSf https://astral.sh/uv/0.9.26/install.sh | sh

# Create the tdp directory
RUN mkdir -p /home/tdp/tdp-dev
Expand Down
2,132 changes: 0 additions & 2,132 deletions poetry.lock

This file was deleted.

41 changes: 24 additions & 17 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,24 +1,31 @@
[tool.poetry]
[project]
name = "tdp-dev"
version = "0.1.0"
description = "tdp-dev dependencies"
authors = ["Stephan Baum <stephan.baume-consultant@dgfip.finances.gouv.fr>"]
authors = [{ name = "TOSIT" }]
license = "Apache-2.0"
readme = "README.md"
homepage = "https://github.com/TOSIT-FR/tdp-dev"
repository = "https://github.com/TOSIT-FR/tdp-dev"
documentation = "https://github.com/TOSIT-FR/tdp-dev"
package-mode = false
requires-python = ">=3.12.0,<4.0"
dependencies = [
"tdp-lib[visualization,mysql,postgresql]",
"jmespath==1.0.1",
"ansible-lint>=26.1.1",
"passlib>=1.7.4",
"pytest-xdist>=3.8.0",
"pytest-testinfra>=10.2.2",
]

[tool.poetry.dependencies]
python = ">=3.9.0,<4.0"
tdp-lib = { path = "tdp-lib", develop = true, extras=["visualization", "mysql", "postgresql-binary"] }
passlib = "1.7.4"
jmespath = "1.0.1"
ansible-lint = {version = "6.17.2", markers = "platform_system != 'Windows'"}
pytest-xdist = "^3.6.1"
pytest-testinfra = "^10.1.1"
[tool.uv]
# https://github.com/ansible/ansible-lint/issues/4348
environments = ["sys_platform != 'win32'"]

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.uv.sources]
tdp-lib = { path = "./tdp-lib", editable = true }

[project.urls]
Homepage = "https://github.com/TOSIT-FR/tdp-dev"
Repository = "https://github.com/TOSIT-FR/tdp-dev"
Documentation = "https://github.com/TOSIT-FR/tdp-dev"

[tool.setuptools]
py-modules = []
2 changes: 1 addition & 1 deletion tdp-lib
Submodule tdp-lib updated 122 files
2 changes: 1 addition & 1 deletion tdp-vagrant
Loading