diff --git a/api/outdated/conftest.py b/api/outdated/conftest.py index 0fc25b0e..9c5be29c 100644 --- a/api/outdated/conftest.py +++ b/api/outdated/conftest.py @@ -25,6 +25,7 @@ register(factories.ReleaseVersionFactory) register(factories.ProjectFactory) register(factories.MaintainerFactory) +register(factories.DependencySourceFactory) register(UserFactory) diff --git a/api/outdated/models.py b/api/outdated/models.py index 990f7f71..3437f3b6 100644 --- a/api/outdated/models.py +++ b/api/outdated/models.py @@ -1,8 +1,7 @@ -from subprocess import run +from __future__ import annotations + from uuid import uuid4 -from django.conf import settings -from django.core.exceptions import ValidationError from django.core.validators import RegexValidator from django.db import models @@ -60,29 +59,11 @@ def pre_save(self, model_instance, add): return super().pre_save(model_instance, add) -def validate_repo_exists(value: str) -> None: - """Validate the existance of a remote git repository.""" - url = "https://" + value - - if value.startswith("file://") and settings.ENV == "test": - url = value - - result = run( - ["/usr/bin/git", "ls-remote", url], - capture_output=True, - check=False, - shell=False, - ) - if result.returncode != 0: - raise ValidationError("Repository does not exist.", params={"value": value}) - - class RepositoryURLField(models.CharField): default_validators = [ RegexValidator( regex=r"^([-_\w]+\.[-._\w]+)\/([-_\w]+)\/([-_\w]+)$", message="Invalid repository url", - ), - validate_repo_exists, + ) ] description = "Field for git repository URLs." diff --git a/api/outdated/outdated/factories.py b/api/outdated/outdated/factories.py index b289e418..5dafdd01 100644 --- a/api/outdated/outdated/factories.py +++ b/api/outdated/outdated/factories.py @@ -54,21 +54,32 @@ class Meta: class ProjectFactory(DjangoModelFactory): name = Faker("uuid4") repo = Sequence(lambda n: "github.com/userorcompany/%s/" % n) + repo_type = "public" + + class Meta: + model = models.Project + + +class DependencySourceFactory(DjangoModelFactory): + project = SubFactory(ProjectFactory) + path = random.choice( + ["/pyproject.toml", "/api/pyproject.toml", "/ember/pnpm-lock.yaml"] + ) @post_generation - def versioned_dependencies(self, create, extracted, **kwargs): + def versions(self, create, extracted, **kwargs): if not create: return # pragma: no cover if extracted: - for versioned_dependency in extracted: - self.versioned_dependencies.add(versioned_dependency) + for version in extracted: + self.versions.add(version) class Meta: - model = models.Project + model = models.DependencySource class MaintainerFactory(DjangoModelFactory): - project = SubFactory(ProjectFactory) + source = SubFactory(DependencySourceFactory) user = SubFactory(UserFactory) class Meta: diff --git a/api/outdated/outdated/migrations/0001_initial.py b/api/outdated/outdated/migrations/0001_initial.py index e2977625..59210187 100644 --- a/api/outdated/outdated/migrations/0001_initial.py +++ b/api/outdated/outdated/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.6 on 2023-10-24 14:19 +# Generated by Django 4.2.6 on 2024-01-12 15:45 from django.db import migrations, models import django.db.models.deletion @@ -8,172 +8,121 @@ class Migration(migrations.Migration): + initial = True dependencies = [ - ("user", "0001_initial"), + ('user', '0001_initial'), ] operations = [ migrations.CreateModel( - name="Dependency", + name='Dependency', fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ("name", models.CharField(max_length=100)), - ( - "provider", - models.CharField( - choices=[("PIP", "PIP"), ("NPM", "NPM")], max_length=10 - ), - ), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=100)), + ('provider', models.CharField(choices=[('PIP', 'PIP'), ('NPM', 'NPM')], max_length=10)), ], options={ - "ordering": ["name", "id"], - "unique_together": {("name", "provider")}, + 'ordering': ['name', 'id'], }, ), migrations.CreateModel( - name="ReleaseVersion", + name='DependencySource', fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ("major_version", models.IntegerField()), - ("minor_version", models.IntegerField()), - ("end_of_life", models.DateField(blank=True, null=True)), - ( - "dependency", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="outdated.dependency", - ), - ), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('path', models.CharField()), ], options={ - "ordering": [ - "end_of_life", - "dependency__name", - "major_version", - "minor_version", - ], - "unique_together": {("dependency", "major_version", "minor_version")}, + 'abstract': False, }, ), migrations.CreateModel( - name="Version", + name='Maintainer', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('is_primary', outdated.models.UniqueBooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='Project', fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ("patch_version", models.IntegerField()), - ("release_date", models.DateField(blank=True, null=True)), - ( - "release_version", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="outdated.releaseversion", - ), - ), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(db_index=True, max_length=100)), + ('repo', outdated.models.RepositoryURLField(max_length=100)), + ('repo_type', models.CharField(choices=[('public', 'public'), ('access-token', 'access-token')], max_length=25)), ], options={ - "ordering": [ - "release_version__end_of_life", - "release_version__dependency__name", - "release_version__major_version", - "release_version__minor_version", - "patch_version", - ], - "unique_together": {("release_version", "patch_version")}, + 'ordering': ['name', 'id'], }, ), migrations.CreateModel( - name="Project", + name='ReleaseVersion', fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ("name", models.CharField(db_index=True, max_length=100)), - ("repo", outdated.models.RepositoryURLField(max_length=100)), - ( - "versioned_dependencies", - models.ManyToManyField(blank=True, to="outdated.version"), - ), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('major_version', models.IntegerField()), + ('minor_version', models.IntegerField()), + ('end_of_life', models.DateField(blank=True, null=True)), + ('dependency', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='outdated.dependency')), ], options={ - "ordering": ["name", "id"], + 'ordering': ['end_of_life', 'dependency__name', 'major_version', 'minor_version'], }, ), migrations.CreateModel( - name="Maintainer", + name='Version', fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ("is_primary", outdated.models.UniqueBooleanField(default=False)), - ( - "project", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="maintainers", - to="outdated.project", - ), - ), - ( - "user", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="user.user" - ), - ), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('patch_version', models.IntegerField()), + ('release_date', models.DateField(blank=True, null=True)), + ('release_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='outdated.releaseversion')), ], + options={ + 'ordering': ['release_version__end_of_life', 'release_version__dependency__name', 'release_version__major_version', 'release_version__minor_version', 'patch_version'], + }, ), migrations.AddConstraint( - model_name="project", - constraint=models.UniqueConstraint( - django.db.models.functions.text.Lower("name"), - name="unique_project_name", - ), + model_name='project', + constraint=models.UniqueConstraint(django.db.models.functions.text.Lower('name'), name='unique_project_name'), ), migrations.AddConstraint( - model_name="project", - constraint=models.UniqueConstraint( - django.db.models.functions.text.Lower("repo"), - name="unique_project_repo", - ), + model_name='project', + constraint=models.UniqueConstraint(django.db.models.functions.text.Lower('repo'), name='unique_project_repo'), + ), + migrations.AddField( + model_name='maintainer', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='maintainers', to='outdated.dependencysource'), + ), + migrations.AddField( + model_name='maintainer', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='user.user'), + ), + migrations.AddField( + model_name='dependencysource', + name='project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sources', to='outdated.project'), + ), + migrations.AddField( + model_name='dependencysource', + name='versions', + field=models.ManyToManyField(blank=True, to='outdated.version'), + ), + migrations.AlterUniqueTogether( + name='dependency', + unique_together={('name', 'provider')}, + ), + migrations.AlterUniqueTogether( + name='version', + unique_together={('release_version', 'patch_version')}, + ), + migrations.AlterUniqueTogether( + name='releaseversion', + unique_together={('dependency', 'major_version', 'minor_version')}, ), migrations.AlterUniqueTogether( - name="maintainer", - unique_together={("user", "project")}, + name='maintainer', + unique_together={('user', 'source')}, ), ] diff --git a/api/outdated/outdated/models.py b/api/outdated/outdated/models.py index 373d46e9..1c6ed1fb 100644 --- a/api/outdated/outdated/models.py +++ b/api/outdated/outdated/models.py @@ -117,11 +117,13 @@ def version(self) -> str: return f"{self.release_version.release_version}.{self.patch_version}" +REPO_TYPES = [(_, _) for _ in ["public", "access-token"]] + + class Project(UUIDModel): name = models.CharField(max_length=100, db_index=True) - - versioned_dependencies = models.ManyToManyField(Version, blank=True) repo = RepositoryURLField(max_length=100) + repo_type = models.CharField(max_length=25, choices=REPO_TYPES) @property def repo_domain(self) -> str: @@ -198,21 +200,34 @@ class Meta: @property def status(self) -> str: - first = self.versioned_dependencies.first() + first = self.sources.all().values_list("versions", flat=True).first() return first.release_version.status if first else STATUS_OPTIONS["undefined"] def __str__(self): return self.name +class DependencySource(UUIDModel): + path = models.CharField() + project = models.ForeignKey( + Project, on_delete=models.CASCADE, related_name="sources" + ) + versions = models.ManyToManyField(Version, blank=True) + + @property + def status(self) -> str: + first = self.versions.first() + return first.release_version.status if first else STATUS_OPTIONS["undefined"] + + class Maintainer(UUIDModel): user = models.ForeignKey(User, on_delete=models.CASCADE) - project = models.ForeignKey( - Project, + source = models.ForeignKey( + DependencySource, on_delete=models.CASCADE, related_name="maintainers", ) - is_primary = UniqueBooleanField(default=False, together=["project"]) + is_primary = UniqueBooleanField(default=False, together=["source"]) class Meta: - unique_together = ("user", "project") + unique_together = ("user", "source") diff --git a/api/outdated/outdated/parser.py b/api/outdated/outdated/parser.py index 2c28a8e0..aa7a0ea5 100644 --- a/api/outdated/outdated/parser.py +++ b/api/outdated/outdated/parser.py @@ -20,8 +20,9 @@ class LockfileParser: """Parse a lockfile and return a list of dependencies.""" - def __init__(self, lockfiles: list[Path]) -> None: + def __init__(self, project: models.Project, lockfiles: list[Path]) -> None: self.lockfiles = lockfiles + self.project = project def _get_provider(self, name: str) -> str: """Get the provider of the lockfile.""" @@ -102,10 +103,8 @@ def _get_release_date(self, version: models.Version) -> date: return parse_date(release_date).date() - def parse(self) -> list[models.Version]: + def parse(self) -> None: """Parse the lockfile and return a dictionary of dependencies.""" - versions = [] - for lockfile in self.lockfiles: name = lockfile.name data = lockfile.read_text() @@ -139,8 +138,9 @@ def parse(self) -> list[models.Version]: and requirements[0][0] in settings.TRACKED_DEPENDENCIES ] - versions.extend( - self._get_version(dependency, provider) for dependency in dependencies + source, _ = models.DependencySource.objects.get_or_create( + path=name, project=self.project + ) + source.versions.set( + [self._get_version(dependency, provider) for dependency in dependencies] ) - - return versions diff --git a/api/outdated/outdated/serializers.py b/api/outdated/outdated/serializers.py index c363eaaf..ab6a2975 100644 --- a/api/outdated/outdated/serializers.py +++ b/api/outdated/outdated/serializers.py @@ -1,6 +1,13 @@ +from django.core.validators import RegexValidator +from rest_framework.validators import UniqueValidator from rest_framework_json_api import serializers from outdated.outdated import models +from outdated.outdated.validators import ( + validate_access_token_required, + validate_no_access_token_when_public, + validate_remote_url, +) from .tracking import Tracker @@ -35,7 +42,7 @@ class Meta: class MaintainerSerializer(serializers.ModelSerializer): included_serializers = { "user": "outdated.user.serializers.UserSerializer", - "project": "outdated.outdated.serializers.ProjectSerializer", + "source": "outdated.outdated.serializers.DependencySourceSerializer", } class Meta: @@ -43,37 +50,80 @@ class Meta: fields = "__all__" -class ProjectSerializer(serializers.ModelSerializer): +class DependencySourceSerializer(serializers.ModelSerializer): maintainers = serializers.ResourceRelatedField( many=True, read_only=True, + ) + + included_serializers = { + "versions": VersionSerializer, + "maintainers": MaintainerSerializer, + } + + class Meta: + model = models.DependencySource + fields = "__all__" + + +class ProjectSerializer(serializers.ModelSerializer): + access_token = serializers.CharField( + max_length=100, + write_only=True, required=False, + allow_blank=True, + validators=[RegexValidator(r"[-_a-zA-Z\d]+")], + ) + + sources = serializers.ResourceRelatedField( + many=True, + read_only=True, + ) + + repo = serializers.CharField( + validators=[ + UniqueValidator(queryset=models.Project.objects.all(), lookup="iexact") + ] + ) + name = serializers.CharField( + validators=[ + UniqueValidator(queryset=models.Project.objects.all(), lookup="iexact") + ] ) included_serializers = { - "versioned_dependencies": "outdated.outdated.serializers.VersionSerializer", - "maintainers": "outdated.outdated.serializers.MaintainerSerializer", + "sources": "outdated.outdated.serializers.DependencySourceSerializer" } class Meta: model = models.Project - fields = ( - "name", - "repo", - "status", - "versioned_dependencies", - "maintainers", - ) - - def create(self, validated_data): + validators = [ + validate_remote_url, + validate_access_token_required, + validate_no_access_token_when_public, + ] + fields = ("name", "repo", "repo_type", "access_token", "status", "sources") + + def create(self, validated_data: dict) -> models.Project: + access_token = None + if "access_token" in validated_data: + access_token = validated_data["access_token"] + del validated_data["access_token"] instance = super().create(validated_data) - Tracker(instance).setup() + Tracker(instance, access_token).setup() return instance def update(self, instance: models.Project, validated_data: dict) -> models.Project: - old_instance = models.Project(repo=instance.repo) + old_instance = models.Project(repo=instance.repo, repo_type=instance.repo_type) + access_token = None + if "access_token" in validated_data: + access_token = validated_data["access_token"] + del validated_data["access_token"] super().update(instance, validated_data) - if instance.clone_path != old_instance.clone_path: + if ( + instance.clone_path != old_instance.clone_path + or instance.repo_type != old_instance.repo_type + ): Tracker(old_instance).delete() - Tracker(instance).setup() + Tracker(instance, access_token).setup() return instance diff --git a/api/outdated/outdated/tests/test_api.py b/api/outdated/outdated/tests/test_api.py index 2c867a5a..4fc9ec39 100644 --- a/api/outdated/outdated/tests/test_api.py +++ b/api/outdated/outdated/tests/test_api.py @@ -120,10 +120,15 @@ def test_version(client, version_factory): @pytest.mark.parametrize("defined", [True, False]) -def test_project(client, project_factory, version_factory, defined): - generated_project = project_factory( - versioned_dependencies=[version_factory()] if defined else [], +def test_project( + client, project_factory, dependency_source_factory, version_factory, defined +): + generated_project = project_factory() + + dependency_source_factory( + versions=[version_factory()] if defined else [], project=generated_project ) + url = reverse("project-list") resp = client.get(url) assert resp.status_code == http_status.HTTP_200_OK @@ -145,22 +150,27 @@ def test_project(client, project_factory, version_factory, defined): == detailed_response_project["repo"] == generated_project.repo ) + assert ( + response_project["repo-type"] + == detailed_response_project["repo-type"] + == generated_project.repo_type + ) if defined: for gen_dep_version, resp_dep_version in zip( - resp_detailed.json()["data"]["relationships"]["versioned-dependencies"][ - "data" - ], - generated_project.versioned_dependencies.all(), + resp_detailed.json()["data"]["relationships"]["sources"]["data"], + generated_project.sources.all(), ): assert gen_dep_version["id"] == str(resp_dep_version.id) else: - assert not generated_project.versioned_dependencies.first() + assert not generated_project.sources.first().versions.first() + assert generated_project.sources.first().status == "UNDEFINED" assert generated_project.status == "UNDEFINED" def test_project_ordered_by_eol( client, project_factory, + dependency_source_factory, release_version_factory, version_factory, ): @@ -174,26 +184,24 @@ def test_project_ordered_by_eol( release_version=release_version_factory(up_to_date=True), ) - project_last = project_factory( - name="A project", - versioned_dependencies=[up_to_date_version], - ) - project_middle = project_factory( - name="B project", - versioned_dependencies=[warning_version], - ) - project_first = project_factory( - name="C project", - versioned_dependencies=[outdated_version], - ) + project_up_to_date = project_factory(name="A project") + dependency_source_factory(versions=[up_to_date_version], project=project_up_to_date) + project_warning = project_factory(name="B project") + dependency_source_factory(versions=[warning_version], project=project_warning) + project_outdated = project_factory(name="C project") + dependency_source_factory(versions=[outdated_version], project=project_outdated) + project_undefined = project_factory(name="D project") url = reverse("project-list") resp = client.get(url) + + assert resp.status_code == http_status.HTTP_200_OK json = resp.json() - assert json["data"][0]["id"] == str(project_first.pk) - assert json["data"][1]["id"] == str(project_middle.pk) - assert json["data"][2]["id"] == str(project_last.pk) + assert json["data"][0]["id"] == str(project_outdated.pk) + assert json["data"][1]["id"] == str(project_warning.pk) + assert json["data"][2]["id"] == str(project_up_to_date.pk) + assert json["data"][3]["id"] == str(project_undefined.pk) def test_maintainer(client, maintainer): @@ -211,15 +219,15 @@ def test_maintainer(client, maintainer): == str(maintainer.user.id) ) assert ( - relationships["project"]["data"]["id"] - == detailed_relationships["project"]["data"]["id"] - == str(maintainer.project.id) + relationships["source"]["data"]["id"] + == detailed_relationships["source"]["data"]["id"] + == str(maintainer.source.id) ) assert ( resp.json()["data"][0]["attributes"] == resp_detailed.json()["data"]["attributes"] ) - assert maintainer.project.maintainers.all()[0] == maintainer + assert maintainer.source.maintainers.all()[0] == maintainer @pytest.mark.django_db(transaction=True) diff --git a/api/outdated/outdated/tests/test_parser.py b/api/outdated/outdated/tests/test_parser.py index 8fc36e54..3e74d2da 100644 --- a/api/outdated/outdated/tests/test_parser.py +++ b/api/outdated/outdated/tests/test_parser.py @@ -2,6 +2,7 @@ import pytest +from outdated.outdated.models import Version from outdated.outdated.parser import LockfileParser from outdated.outdated.tracking import Tracker @@ -110,9 +111,9 @@ def test_parser(db, tmp_repo_root, project, lockfile, content, expected): assert len(lockfiles) == 1 assert lockfiles[0].name == lockfile - results = LockfileParser(lockfiles).parse() + LockfileParser(project, lockfiles).parse() - assert len(results) == len(expected) + assert len(project.sources.values_list("versions", flat=True)) == len(expected) - for result in results: - assert str(result) in expected + for result in project.sources.values_list("versions", flat=True): + assert str(Version.objects.get(id=result)) in expected diff --git a/api/outdated/outdated/tests/test_tracking.py b/api/outdated/outdated/tests/test_tracking.py index a30b6ad0..e10ac490 100644 --- a/api/outdated/outdated/tests/test_tracking.py +++ b/api/outdated/outdated/tests/test_tracking.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from unittest.mock import PropertyMock, call import pytest @@ -12,13 +14,14 @@ @pytest.mark.parametrize( - "repo,reinitialize", + "repo,repo_type,access_token,reinitialize", [ - ("github.com/Adfinis/Outdated", False), - ("github.com/adfinis/outdated", False), - ("Github.Com/ADFINIS/OutdAted", False), - ("github.com/adfinis/mysagw", True), - ("github.com/adfinis/timed-frontend", True), + ("github.com/Adfinis/Outdated", "public", None, False), + ("github.com/adfinis/outdated", "access-token", "token", False), + ("Github.Com/ADFINIS/OutdAted", "public", None, False), + ("github.com/adfinis/mysagw", "public", None, True), + ("github.com/adfinis/timed-frontend", "public", None, True), + ("github.com/adfinis/timed-backend", "public", "token", True), ], ) def test_serializer_patch( @@ -28,11 +31,17 @@ def test_serializer_patch( tracker_mock, repo, reinitialize, + repo_type, + settings, + access_token, ): - project = project_factory(repo="github.com/adfinis/outdated") + project = project_factory(repo="github.com/adfinis/outdated", repo_type=repo_type) setup_mock = tracker_mock("setup") delete_mock = tracker_mock("delete") + # don't depend on github + settings.VALIDATE_REMOTES = False + data = { "data": { "type": "projects", @@ -40,11 +49,15 @@ def test_serializer_patch( "attributes": { "name": project.name, "repo": repo, + "repo_type": "access-token" if access_token else "public", }, "relationships": {}, }, } + if access_token: + data["data"]["attributes"]["access_token"] = access_token + url = reverse("project-detail", args=[project.id]) resp = client.patch(url, data) @@ -58,16 +71,22 @@ def test_serializer_patch( tracker_init_mock.call_args_list[0].args[0].repo == "github.com/adfinis/outdated" ) - tracker_init_mock.assert_called_with(project) + tracker_init_mock.assert_called_with(project, access_token) else: delete_mock.assert_not_called() setup_mock.assert_not_called() tracker_init_mock.assert_not_called() -def test_serializer_create(client, project_factory, tracker_init_mock, tracker_mock): +@pytest.mark.parametrize("access_token", [None, "token"]) +def test_serializer_create( + client, tracker_init_mock, tracker_mock, settings, access_token +): setup_mock = tracker_mock("setup") + # don't depend on github + settings.VALIDATE_REMOTES = False + data = { "data": { "type": "projects", @@ -75,17 +94,21 @@ def test_serializer_create(client, project_factory, tracker_init_mock, tracker_m "attributes": { "name": "foo", "repo": "github.com/adfinis/outdated", + "repo_type": "access-token" if access_token else "public", }, "relationships": {}, }, } + if access_token: + data["data"]["attributes"]["access_token"] = access_token + url = reverse("project-list") response = client.post(url, data) assert response.status_code == status.HTTP_201_CREATED project = Project.objects.get(name="foo") - tracker_init_mock.assert_called_once_with(project) + tracker_init_mock.assert_called_once_with(project, access_token) setup_mock.assert_called_once() @@ -99,11 +122,16 @@ def test_view_delete(client, project, tracker_init_mock, tracker_mock): assert not Project.objects.filter(id=project.id) -def test_clone(db, project_factory, tmp_repo_root, tracker_mock): +@pytest.mark.parametrize("access_token", [None, "token"]) +def test_clone(db, project_factory, tmp_repo_root, tracker_mock, access_token): tracker_run_mock = tracker_mock("_run") - project: Project = project_factory(repo="github.com/adfinis/outdated") + tracker_delete_mock = tracker_mock("delete") + project: Project = project_factory( + repo="github.com/adfinis/outdated", + repo_type="access-token" if access_token else "public", + ) - tracker = Tracker(project) + tracker = Tracker(project, access_token) tracker.clone() @@ -117,7 +145,9 @@ def test_clone(db, project_factory, tmp_repo_root, tracker_mock): "--depth=1", "--filter=tree:0", "--single-branch", - "https://github.com/adfinis/outdated", + "https://outdated:token@github.com/adfinis/outdated" + if access_token + else "https://github.com/adfinis/outdated", tmp_repo_root / "github.com/adfinis/outdated", ], ), @@ -133,6 +163,8 @@ def test_clone(db, project_factory, tmp_repo_root, tracker_mock): ], ) + tracker_delete_mock.assert_called_once() + @pytest.mark.parametrize("requires_local_copy", [True, False]) @pytest.mark.parametrize("exists", [True, False]) @@ -184,6 +216,7 @@ def test_sync( exists, mocker, version_factory, + dependency_source_factory, ): project_path = tmp_repo_root / project.clone_path @@ -203,15 +236,17 @@ def test_sync( ) versions = version_factory.create_batch(5) + + def side_effect() -> None: + dependency_source_factory(versions=versions, project=project) + lockfile_parser_parser_mock = mocker.patch.object( - LockfileParser, - "parse", - return_value=versions, + LockfileParser, "parse", side_effect=side_effect ) tracker = Tracker(project) assert tracker.local_path == project_path - assert not project.versioned_dependencies.all() + assert not project.sources.all() if exists: project_path.mkdir(parents=True, exist_ok=False) @@ -224,13 +259,13 @@ def test_sync( tracker_checkout_mock.assert_called_once() - lockfile_parser_init_mock.assert_called_once_with([]) + lockfile_parser_init_mock.assert_called_once_with(project, []) lockfile_parser_parser_mock.assert_called_once_with() tracker_lockfile_mock.assert_called_once() - assert set(project.versioned_dependencies.all()) == set(versions) + assert set(project.sources.first().versions.all()) == set(versions) @pytest.mark.parametrize("exists", [True, False]) diff --git a/api/outdated/outdated/tests/test_validators.py b/api/outdated/outdated/tests/test_validators.py new file mode 100644 index 00000000..c90698fa --- /dev/null +++ b/api/outdated/outdated/tests/test_validators.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from contextlib import suppress +from subprocess import run +from typing import TYPE_CHECKING +from unittest.mock import call + +import pytest +from django.core.exceptions import ValidationError + +from outdated.outdated import validators +from outdated.outdated.serializers import ProjectSerializer + +if TYPE_CHECKING: + from pathlib import Path + from typing import Literal + + from pytest_mock import MockerFixture + + from outdated.outdated.factories import ProjectFactory + from outdated.outdated.models import Project + +type RepoType = Literal["public", "access-token"] + + +@pytest.mark.django_db() +@pytest.mark.parametrize("access_token", [None, "token"]) +@pytest.mark.parametrize("repo_type", ["public", "access-token"]) +@pytest.mark.parametrize("instance", [True, False]) +def test_access_token_required( + repo_type: RepoType, access_token: None | str, instance: bool, project: Project +) -> None: + attrs = { + "repo_type": repo_type, + "access_token": access_token, + } + + serializer = ProjectSerializer(project if instance else None) + + try: + validators.validate_access_token_required(attrs, serializer) + if repo_type == "access-token" and not access_token: + assert serializer.instance + elif repo_type == "access-token": + assert access_token + else: + assert repo_type == "public" + + except ValidationError: + assert repo_type == "access-token" + assert not access_token + + +@pytest.mark.parametrize("access_token", [None, "token"]) +@pytest.mark.parametrize("repo_type", ["public", "access-token"]) +def test_no_access_token_when_public( + repo_type: RepoType, access_token: None | str +) -> None: + attrs = { + "repo_type": repo_type, + "access_token": access_token, + } + + try: + validators.validate_no_access_token_when_public(attrs, ProjectSerializer()) + except ValidationError: + assert repo_type == "public" + assert access_token + + +@pytest.mark.parametrize("access_token", [None, "token"]) +@pytest.mark.parametrize("is_public", [True, False]) +@pytest.mark.parametrize("repo_type", ["public", "access-token"]) +@pytest.mark.parametrize("remote_exists", [True, False]) +def test_remote_exists( # noqa: C901 + repo_type: str, + access_token: str | None, + is_public: bool, + remote_exists: bool, + mocker: MockerFixture, +) -> None: + def side_effect(url: str) -> bool: + if not remote_exists: + return False + if repo_type == "public": + return True + return is_public or "@" in url + + check_remote_existance_mock = mocker.patch.object( + validators, "check_remote_existance", side_effect=side_effect + ) + + attrs = { + "repo": "my.git.com/foo/bar", + "repo_type": repo_type, + } + + if access_token: + attrs["access_token"] = access_token + + try: + validators.validate_remote_url(attrs, ProjectSerializer()) + if repo_type == "access-token" and access_token: + check_remote_existance_mock.assert_has_calls( + [call("outdated:token@my.git.com/foo/bar"), call("my.git.com/foo/bar")] + ) + assert remote_exists + return + if ( + repo_type == "access-token" + and not access_token + or repo_type == "public" + and access_token + ): + check_remote_existance_mock.assert_not_called() + return + check_remote_existance_mock.assert_called_once() + assert remote_exists + except ValidationError as e: + if not remote_exists: + assert e.message == "Repository does not exist." # noqa: PT017 + return + assert e.message == "Repository is public." # noqa: PT017 + assert is_public + assert access_token + assert repo_type == "access-token" + + +@pytest.mark.django_db() +@pytest.mark.parametrize("repo", ["my.git.com/foo/bar", "other.git.com/foo/bar"]) +@pytest.mark.parametrize("repo_type", ["public", "access-token"]) +def test_remote_exists_with_instance( + repo_type: RepoType, + repo: str, + mocker: MockerFixture, + project_factory: ProjectFactory, +) -> None: + check_remote_existance_mock = mocker.patch.object( + validators, "check_remote_existance" + ) + + project = project_factory(repo="my.git.com/foo/bar", repo_type="public") + + attrs = {"repo": repo, "repo_type": repo_type} + + if repo_type == "access-token": + attrs["access_token"] = "token" # noqa: S105 + + serializer = ProjectSerializer(project) + + with suppress(ValidationError): + validators.validate_remote_url(attrs, serializer) + + if project.repo == repo and project.repo_type == repo_type: + check_remote_existance_mock.assert_not_called() + else: + check_remote_existance_mock.assert_called() + + +@pytest.mark.parametrize( + "exists", + [True, False], +) +def test_check_remote_exists(exists: bool, tmp_repo_root: Path) -> None: + path = tmp_repo_root.absolute() / "project" + url = f"file://{path.absolute()}" + if exists: + path.mkdir() + run(["/usr/bin/git", "init"], shell=False, check=False, cwd=path) + assert validators.check_remote_existance(url) == exists diff --git a/api/outdated/outdated/tracking.py b/api/outdated/outdated/tracking.py index c6eab1a0..eb24fcb4 100644 --- a/api/outdated/outdated/tracking.py +++ b/api/outdated/outdated/tracking.py @@ -20,8 +20,9 @@ class RepoError(ValueError): class Tracker: - def __init__(self, project: Project) -> None: + def __init__(self, project: Project, access_token: str | None = None) -> None: self.project = project + self.access_token = access_token self.local_path = Path(f"{settings.REPOSITORY_ROOT}/{self.project.clone_path}") def _run( @@ -42,6 +43,11 @@ def _run( def clone(self): self.delete() + url = ( + "https://" + + (f"outdated:{self.access_token}@" if self.access_token else "") + + self.project.repo + ) self._run( [ "git", @@ -50,7 +56,7 @@ def clone(self): "--depth=1", "--filter=tree:0", "--single-branch", - "https://" + self.project.repo, + url, self.local_path.absolute(), ], ) @@ -95,15 +101,14 @@ def sync(self): if not self.local_path.exists(): self.clone() self.checkout() - dependencies = LockfileParser(self.lockfiles).parse() - self.project.versioned_dependencies.set(dependencies) + LockfileParser(self.project, self.lockfiles).parse() def setup(self): # pragma: no cover self.clone() self.checkout() self.sync() - def delete(self): + def delete(self): # pragma: no cover rmtree(self.local_path, True) self._run( [ diff --git a/api/outdated/outdated/validators.py b/api/outdated/outdated/validators.py new file mode 100644 index 00000000..703b0233 --- /dev/null +++ b/api/outdated/outdated/validators.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from subprocess import run +from typing import TYPE_CHECKING + +from django.conf import settings +from django.core.exceptions import ValidationError + +from outdated.validators import with_context + +if TYPE_CHECKING: + from typing import Any + + from outdated.outdated.serializers import ProjectSerializer + + +def check_remote_existance(url: str) -> bool: + """Validate the existance of a remote git repository.""" + remote_url = ( + "" if url.startswith("file://") and settings.ENV == "test" else "https://" + ) + url + + result = run( + ["/usr/bin/git", "ls-remote", remote_url], + capture_output=True, + check=False, + shell=False, + ) + + return not result.returncode + + +@with_context +def validate_access_token_required( + attrs: dict[str, Any], serializer: ProjectSerializer +) -> None: + if serializer.instance: + return + + if attrs["repo_type"] == "public": + return + if not attrs.get("access_token"): + raise ValidationError("Access Token is required.", "required") + + +@with_context +def validate_no_access_token_when_public( + attrs: dict[str, Any], serializer: ProjectSerializer +) -> None: + if attrs["repo_type"] == "access-token": + return + if attrs.get("access_token"): + raise ValidationError( + "Access Token is not valid for public repositories.", + params={"value": attrs.get("access_token")}, + ) + + +@with_context +def validate_remote_url(attrs: dict[str, Any], serializer: ProjectSerializer) -> None: # noqa: C901 + if ( + not settings.VALIDATE_REMOTES + or (instance := serializer.instance) + and instance.repo == attrs["repo"] + and instance.repo_type == attrs["repo_type"] + ): + return + url = attrs["repo"] + if attrs["repo_type"] == "access-token": + # other validator will handle this + if not (access_token := attrs.get("access_token")): + return + url = f"outdated:{access_token}@" + url + # other validator will handle this + elif attrs.get("access_token"): + return + + if not check_remote_existance(url): + raise ValidationError("Repository does not exist.", params={"value": url}) + + if attrs["repo_type"] == "access-token" and check_remote_existance(attrs["repo"]): + raise ValidationError("Repository is public.", params={"value": url}) diff --git a/api/outdated/outdated/views.py b/api/outdated/outdated/views.py index 43f155df..36820b69 100644 --- a/api/outdated/outdated/views.py +++ b/api/outdated/outdated/views.py @@ -1,7 +1,7 @@ from django.db.models import Max from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.viewsets import ModelViewSet +from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from . import models, serializers from .tracking import Tracker @@ -11,7 +11,7 @@ class ProjectViewSet(ModelViewSet): queryset = ( models.Project.objects.all() .annotate( - latest_eol=Max("versioned_dependencies__release_version__end_of_life"), + latest_eol=Max("sources__versions__release_version__end_of_life"), ) .order_by("latest_eol") ) @@ -45,6 +45,10 @@ class DependencyViewSet(ModelViewSet): serializer_class = serializers.DependencySerializer +class DependencySourceViewSet(ReadOnlyModelViewSet): + queryset = models.DependencySource.objects.all() + + class MaintainerViewset(ModelViewSet): queryset = models.Maintainer.objects.all() serializer_class = serializers.MaintainerSerializer diff --git a/api/outdated/settings.py b/api/outdated/settings.py index d5b2e558..edbae018 100644 --- a/api/outdated/settings.py +++ b/api/outdated/settings.py @@ -170,9 +170,6 @@ def default(default_dev=env.NOTSET, default_prod=env.NOTSET): JSON_API_FORMAT_TYPES = "dasherize" JSON_API_PLURALIZE_TYPES = True -# Github API -GITHUB_API_TOKEN = env.str("GITHUB_API_TOKEN") - # Syncproject settings TRACKED_DEPENDENCIES = env.list( "TRACKED_DEPENDENCIES", @@ -193,3 +190,6 @@ def default(default_dev=env.NOTSET, default_prod=env.NOTSET): PYPI_FILES = ["poetry.lock"] SUPPORTED_LOCK_FILES = [*NPM_FILES, *PYPI_FILES] + +# Variables used only in testing +VALIDATE_REMOTES = True diff --git a/api/outdated/tests/test_repo_field.py b/api/outdated/tests/test_repo_field.py deleted file mode 100644 index d27b6db7..00000000 --- a/api/outdated/tests/test_repo_field.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from subprocess import run -from typing import TYPE_CHECKING - -import pytest -from django.core.exceptions import ValidationError - -from outdated.models import validate_repo_exists - -if TYPE_CHECKING: - from pathlib import Path - - -@pytest.mark.parametrize( - "exists", - [True, False], -) -def test_repository_exists_validator(exists: bool, tmp_repo_root: Path) -> None: - path = tmp_repo_root.absolute() / "project" - url = f"file://{path.absolute()}" - if exists: - path.mkdir() - run(["/usr/bin/git", "init"], shell=False, check=False, cwd=path) - try: - validate_repo_exists(url) - assert exists - except ValidationError: - assert not exists diff --git a/api/outdated/tests/test_unique_boolean_field.py b/api/outdated/tests/test_unique_boolean_field.py index d028fb8d..a7880a9d 100644 --- a/api/outdated/tests/test_unique_boolean_field.py +++ b/api/outdated/tests/test_unique_boolean_field.py @@ -6,7 +6,7 @@ def test_unique_boolean_field(db, maintainer_factory): assert Maintainer.objects.count() == 1 assert maintainer.is_primary - other_maintainer = maintainer_factory(project=maintainer.project) + other_maintainer = maintainer_factory(source=maintainer.source) assert Maintainer.objects.count() == 2 assert not other_maintainer.is_primary diff --git a/api/outdated/validators.py b/api/outdated/validators.py new file mode 100644 index 00000000..8e01d855 --- /dev/null +++ b/api/outdated/validators.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + from typing import Any + + from rest_framework_json_api.serializers import Serializer + +type ValidatorFunction = Callable[[dict[str, Any], Serializer], None] + + +def with_context(func: ValidatorFunction) -> ValidatorFunction: + def wrapper(attrs: dict[str, Any], serializer: Serializer) -> None: + func(attrs, serializer) + + wrapper.requires_context = True + return wrapper diff --git a/ember/app/components/form.hbs b/ember/app/components/form.hbs index 6a8fadff..d94c62b7 100644 --- a/ember/app/components/form.hbs +++ b/ember/app/components/form.hbs @@ -8,7 +8,10 @@ model=@model loading=this.loading input=(component - "validated-input" model=@model submitted=this.submitted + "validated-input" + model=@model + submitted=this.submitted + validateModel=this.validateModel ) button=(component "uk-button" label="Save" type="submit") ) diff --git a/ember/app/components/project-form/component.js b/ember/app/components/project-form/component.js index 49d3d27b..6a169384 100644 --- a/ember/app/components/project-form/component.js +++ b/ember/app/components/project-form/component.js @@ -21,9 +21,7 @@ export default class ProjectFormComponent extends Component { constructor(...args) { super(...args); - if (this.args.project) { - scheduleOnce('actions', this, 'initUsers'); - } + if (this.args.project) scheduleOnce('actions', this, 'initUsers'); } initUsers() { @@ -36,6 +34,10 @@ export default class ProjectFormComponent extends Component { saveProject = dropTask(async () => { try { + if (this.project.repoType === 'public') { + this.project.accessToken = ''; + } + const project = await this.project.save({ adapterOptions: { include: @@ -63,6 +65,7 @@ export default class ProjectFormComponent extends Component { }); this.router.transitionTo('projects.detailed', project.id); + this.project.accessToken = ''; this.notification.success('Successfully saved!'); } catch (e) { this.notification.danger(e); @@ -81,4 +84,8 @@ export default class ProjectFormComponent extends Component { get users() { return this.store.peekAll('user'); } + + get repoTypes() { + return ['public', 'access-token']; + } } diff --git a/ember/app/components/project-form/template.hbs b/ember/app/components/project-form/template.hbs index 9eabb803..cfdcc972 100644 --- a/ember/app/components/project-form/template.hbs +++ b/ember/app/components/project-form/template.hbs @@ -11,7 +11,25 @@ as |f| > - +
+ +
+ + +
+
+ + \ No newline at end of file +{{#unless @hidden}} + +{{/unless}} \ No newline at end of file diff --git a/ember/app/components/validated-input/render/component.js b/ember/app/components/validated-input/render/component.js index fe431b55..e0e93719 100644 --- a/ember/app/components/validated-input/render/component.js +++ b/ember/app/components/validated-input/render/component.js @@ -1,9 +1,17 @@ +import { scheduleOnce } from '@ember/runloop'; import Component from '@glimmer/component'; import getMessages from 'ember-changeset-validations/utils/get-messages'; import PowerSelect from 'ember-power-select/components/power-select'; import PowerSelectMultiple from 'ember-power-select/components/power-select-multiple'; export default class RenderComponent extends Component { + constructor(...args) { + super(...args); + + if (typeof this.args.hidden === 'boolean') { + scheduleOnce('actions', this.args, 'validateModel', this.args.model); + } + } get selectComponent() { return this.args.multiple ? PowerSelectMultiple : PowerSelect; } diff --git a/ember/app/components/validated-input/render/template.hbs b/ember/app/components/validated-input/render/template.hbs index b13d4c33..14ea34d6 100644 --- a/ember/app/components/validated-input/render/template.hbs +++ b/ember/app/components/validated-input/render/template.hbs @@ -1,9 +1,5 @@ -
- {{#if (or @label (not @placeholder))}} - <@labelComponent @label={{or @label this.name}} /> - {{/if}} +{{#if @raw}}
- {{#if (eq @type "select")}}
-
\ No newline at end of file +{{else}} +
+ {{#if (and (or @label (not @placeholder)))}} + <@labelComponent @label={{or @label this.name}} /> + {{/if}} +
+ + {{#if (eq @type "select")}} + + {{else if (eq @type "date")}} + + {{else}} + + {{/if}} + <@errorComponent /> +
+
+{{/if}} \ No newline at end of file diff --git a/ember/app/components/validated-input/types/select/template.hbs b/ember/app/components/validated-input/types/select/template.hbs index 9989b2c9..5d5b6f00 100644 --- a/ember/app/components/validated-input/types/select/template.hbs +++ b/ember/app/components/validated-input/types/select/template.hbs @@ -1,5 +1,5 @@ <@selectComponent - @searchEnabled={{true}} + @searchEnabled={{@searchable}} @options={{@options}} @selected={{@value}} @onChange={{this.onUpdate}} diff --git a/ember/app/models/project.js b/ember/app/models/project.js index 3a5ce0a0..6fea72ad 100644 --- a/ember/app/models/project.js +++ b/ember/app/models/project.js @@ -5,6 +5,9 @@ export default class ProjectModel extends Model { @attr name; @attr status; @attr repo; + @attr({ defaultValue: 'public' }) repoType; + @attr accessToken; + @hasMany('version', { inverse: null, async: false }) versionedDependencies; @hasMany('maintainer', { inverse: 'project', async: false }) maintainers; diff --git a/ember/app/styles/components/_form.scss b/ember/app/styles/components/_form.scss index 9f0a7cc9..f3e0210b 100644 --- a/ember/app/styles/components/_form.scss +++ b/ember/app/styles/components/_form.scss @@ -7,3 +7,17 @@ .is-invalid:focus { border: $form-border-width solid $form-danger-color; } + +.repo-inputs { + display: flex; + + & > div:first-of-type { + width: 90%; + } + + & > div:last-of-type { + min-width: 8rem; + display: grid; + align-content: start; + } +} diff --git a/ember/app/utils/validations-when-other.js b/ember/app/utils/validations-when-other.js new file mode 100644 index 00000000..3f3e05e8 --- /dev/null +++ b/ember/app/utils/validations-when-other.js @@ -0,0 +1,15 @@ +import validateWhenOther from 'outdated/validators/validate-when-other'; + +const validationsWhenOther = ({ + field, + otherFieldValidator, + fieldValidators, +}) => + fieldValidators.map((fv) => + validateWhenOther({ + field, + otherFieldValidator, + fieldValidator: fv, + }), + ); +export default validationsWhenOther; diff --git a/ember/app/validations/project.js b/ember/app/validations/project.js index b03c1e0a..e65e5076 100644 --- a/ember/app/validations/project.js +++ b/ember/app/validations/project.js @@ -2,16 +2,31 @@ import { validatePresence, validateLength, validateFormat, + validateInclusion, } from 'ember-changeset-validations/validators'; +import validationsWhenOther from 'outdated/utils/validations-when-other'; + export default { - name: [validatePresence(true), validateLength({ max: 100 })], + name: [ + validatePresence({ presence: true, ignoreBlank: true }), + validateLength({ max: 100 }), + ], + repoType: [validatePresence(true)], repo: [ - validatePresence(true), + validatePresence({ presence: true, ignoreBlank: true }), validateLength({ max: 200 }), validateFormat({ regex: /^([-_\w]+\.[-._\w]+)\/([-_\w]+)\/([-_\w]+)$/, }), ], + accessToken: validationsWhenOther({ + field: 'repoType', + otherFieldValidator: validateInclusion({ in: ['access-token'] }), + fieldValidators: [ + validatePresence({ presence: true, ignoreBlank: true }), + validateFormat({ regex: /^[-_a-zA-Z\d]*$/ }), + ], + }), maintainers: [], }; diff --git a/ember/app/validators/validate-when-other.js b/ember/app/validators/validate-when-other.js new file mode 100644 index 00000000..c05127ec --- /dev/null +++ b/ember/app/validators/validate-when-other.js @@ -0,0 +1,13 @@ +import { get } from '@ember/object'; + +export default function validateWhenOther(options = {}) { + return (key, value, oldValue, changes, content) => { + const { field, otherFieldValidator, fieldValidator } = options; + const fieldValue = get(changes, field) || get(content, field); + + const result = otherFieldValidator(field, fieldValue, null, null, null); + if (result !== true) return true; + + return fieldValidator(key, value, oldValue, changes, content); + }; +} diff --git a/ember/mirage/factories/project.js b/ember/mirage/factories/project.js index ecdbdbd5..500b5ad6 100644 --- a/ember/mirage/factories/project.js +++ b/ember/mirage/factories/project.js @@ -4,6 +4,7 @@ import { Factory, trait } from 'miragejs'; export default Factory.extend({ name: () => `${faker.hacker.adjective()} ${faker.company.bsNoun()}`, status: () => 'UNDEFINED', + repoType: () => faker.helpers.arrayElement(['public', 'access-token']), repo() { return `github.com/${faker.internet.domainWord()}/${faker.helpers.slugify( diff --git a/ember/tests/integration/components/form-test.js b/ember/tests/integration/components/form-test.js index f674cc4e..17c5c722 100644 --- a/ember/tests/integration/components/form-test.js +++ b/ember/tests/integration/components/form-test.js @@ -30,7 +30,7 @@ module('Integration | Component | form', function (hooks) { await render( hbs`
`, ); - assert.dom('form input.ember-power-select-trigger-multiple-input').exists(); + assert.dom('form div.ember-power-select-multiple-trigger').exists(); }); test('it renders errors', async function (assert) {