feat: Update from origin

.gitattributes

@@ -0,0 +1,11 @@
+* text=auto eol=lf
+
+*.json text
+*.md text
+*.py text
+*.svg text
+*.txt text
+*.yml text
+*.yaml text
+
+*.png binary

.github/workflows/scripts/icon_path_validator.py

@@ -2,13 +2,19 @@
 import pathlib
 import string
 from typing import Final
+from xml.etree import ElementTree
 
 
 class InvalidStructureException(Exception):
     def __init__(self, msg: str):
         super().__init__(msg)
 
 
+class InvalidSVGException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
 class IconValidator:
     def __init__(self):
         self._SOURCE_DIR: Final[str] = "src"
@@ -23,6 +29,16 @@ def validate(self) -> None:
         for file in self._FULL_IMAGES_DIR.iterdir():
             if file.name not in json_icons:
                 raise InvalidStructureException(f"{file.name} must be used, {file} isn't used!")
+            if file.name.lower().endswith(".svg"):
+                self._validate_svg(file)
+
+    def _validate_svg(self, file: pathlib.Path) -> None:
+        try:
+            with file.open("r", encoding="utf8") as f:
+                content: str = f.read()
+            ElementTree.fromstring(content)
+        except ElementTree.ParseError as e:
+            raise InvalidSVGException(f"Invalid SVG '{file.name}': {e}")
 
     def get_json_icons(self) -> set[str]:
         letters: list[str] = list(string.ascii_lowercase)

.github/workflows/scripts/schema_validator.py

@@ -0,0 +1,38 @@
+import json
+import pathlib
+from typing import Final
+
+from jsonschema import validate, ValidationError
+
+
+class SchemaValidationException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class SchemaValidator:
+    def __init__(self):
+        self._SOURCE_DIR: Final[str] = "src"
+        self._TECH_DIR: Final[str] = "technologies"
+        self._FULL_TECH_DIR: Final[pathlib.Path] = pathlib.Path(self._SOURCE_DIR).joinpath(self._TECH_DIR)
+        self._SCHEMA_FILE: Final[pathlib.Path] = pathlib.Path("schema.json")
+
+    def validate(self) -> None:
+        if not self._SCHEMA_FILE.is_file():
+            raise FileNotFoundError(f"Schema file '{self._SCHEMA_FILE}' not found!")
+        with self._SCHEMA_FILE.open("r", encoding="utf8") as f:
+            schema: dict = json.load(f)
+        for tech_file in sorted(self._FULL_TECH_DIR.iterdir()):
+            if not tech_file.name.endswith(".json"):
+                continue
+            with tech_file.open("r", encoding="utf8") as f:
+                technologies: dict = json.load(f)
+            try:
+                validate(instance=technologies, schema=schema)
+            except ValidationError as e:
+                path: str = " -> ".join(str(p) for p in e.absolute_path) if e.absolute_path else "root"
+                raise SchemaValidationException(f"{tech_file.name}: {e.message} (at {path})")
+
+
+if __name__ == '__main__':
+    SchemaValidator().validate()

.github/workflows/scripts/technology_validator.py

@@ -74,6 +74,16 @@ def __init__(self, msg: str):
         super().__init__(msg)
 
 
+class TechNotFoundException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class InvalidURLException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
 class AbstractValidator:
     def __init__(self, required: bool = False):
         self._required = required
@@ -185,6 +195,27 @@ def get_type(self) -> list[Type]:
         return [str]
 
 
+class URLValidator(StringValidator):
+    def __init__(self, required: bool = False):
+        super().__init__(required)
+        self._url_pattern: Final[re.Pattern] = re.compile(
+            r"^https?://"
+            r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+"
+            r"[A-Za-z0-9-]{2,}"
+            r"(?::\d+)?"
+            r"(?:/[^\s]*)?"
+            r"$"
+        )
+
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        if not self._url_pattern.match(data):
+            self._set_custom_error(InvalidURLException(f"Tech '{tech_name}' has invalid URL: '{data}'"))
+            return False
+        return True
+
+
 class BoolValidator(AbstractValidator):
     def get_type(self) -> list[Type]:
         return [bool]
@@ -200,6 +231,26 @@ def get_type(self) -> list[Type]:
         return [dict]
 
 
+class DNSValidator(DictValidator):
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        for k, v in data.items():
+            if not isinstance(k, str):
+                self._set_custom_error(InvalidKeyException(f"key in DNS for tech '{tech_name}' has an invalid type. 'str' is required, got type '{type(k).__name__}' -> '{k}'"))
+                return False
+            if not isinstance(v, list):
+                self._set_custom_error(InvalidKeyException(f"value in DNS for tech '{tech_name}' has an invalid type. 'list' is required, got type '{type(v).__name__}' -> '{v}'"))
+                return False
+            for record in v:
+                if not isinstance(record, str):
+                    self._set_custom_error(InvalidTypeForFieldException(f"Invalid type for dns in tech '{tech_name}', selector '{v}' '{record}' key must be string!"))
+                    return False
+                if not self._validate_regex(tech_name, record):
+                    return False
+        return True
+
+
 class CategoryValidator(ArrayValidator):
     def __init__(self, categories: list[int], required: bool = False):
         super().__init__(required)
@@ -297,6 +348,22 @@ def _validate(self, tech_name: str, data: Any) -> bool:
         return True
 
 
+class ReferenceValidator(ArrayValidator):
+    def __init__(self, all_techs: set[str]):
+        super().__init__()
+        self._all_techs: Final[set[str]] = all_techs
+
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        for ref in data:
+            clean_ref: str = ref.split(r"\;")[0]
+            if clean_ref not in self._all_techs:
+                self._set_custom_error(TechNotFoundException(f"Tech '{tech_name}' references '{clean_ref}' but it doesn't exist!"))
+                return False
+        return True
+
+
 class TechnologiesValidator:
     def __init__(self, file_name: str):
         self._SOURCE_DIR: Final[str] = "src"
@@ -308,22 +375,23 @@ def __init__(self, file_name: str):
         self._IMAGES_DIR: Final[str] = "images"
         self._ICONS_DIR: Final[str] = "icons"
         self._ICONS: Final[list[str]] = [icon.name for icon in pathlib.Path(self._SOURCE_DIR).joinpath(self._IMAGES_DIR).joinpath(self._ICONS_DIR).iterdir()]
+        self._ALL_TECHS: Final[set[str]] = self._get_all_tech_names()
         self._validators: dict[str, AbstractValidator] = {  # TODO confidence and version validator
             "cats": CategoryValidator(self._CATEGORIES, True),
-            "website": StringValidator(True),
+            "website": URLValidator(True),
             "description": StringValidator(),
             "icon": IconValidator(self._ICONS),
             "cpe": CPEValidator(),
             "saas": BoolValidator(),
             "oss": BoolValidator(),
             "pricing": PricingValidator(),
-            "implies": ArrayValidator(),  # TODO cat validation
-            "requires": ArrayValidator(),  # TODO ^
-            "excludes": ArrayValidator(),  # TODO ^
+            "implies": ReferenceValidator(self._ALL_TECHS),
+            "requires": ReferenceValidator(self._ALL_TECHS),
+            "excludes": ReferenceValidator(self._ALL_TECHS),
             "requiresCategory": CategoryValidator(self._CATEGORIES),
             "cookies": DictValidator(contains_regex=True),
             "dom": DomValidator(),
-            "dns": DictValidator(contains_regex=True),
+            "dns": DNSValidator(contains_regex=True),
             "js": DictValidator(contains_regex=True),
             "headers": DictValidator(contains_regex=True),
             "text": ArrayValidator(contains_regex=True),
@@ -365,6 +433,16 @@ def _duplicate_key_validator(cls, pairs: list[tuple[str, Any]]) -> dict[str, Any
             result[key] = value
         return result
 
+    def _get_all_tech_names(self) -> set[str]:
+        all_techs: set[str] = set()
+        for letter in list(string.ascii_lowercase) + ["_"]:
+            tech_file: pathlib.Path = self._FULL_TECH_DIR.joinpath(f"{letter}.json")
+            if tech_file.exists():
+                with tech_file.open("r", encoding="utf8") as f:
+                    technologies: dict = json.load(f)
+                all_techs.update(technologies.keys())
+        return all_techs
+
 
 class TechnologyProcessor:
     def __init__(self, tech_name: str, tech_data: dict, validators: dict[str, AbstractValidator]):
@@ -389,4 +467,4 @@ def process(self) -> None:
 if __name__ == '__main__':
     # for letter in string.ascii_lowercase + "_":
     #     TechnologiesValidator(os.getenv("TECH_FILE_NAME", f"{letter}.json")).validate()
-    TechnologiesValidator(os.getenv("TECH_FILE_NAME", f"a.json")).validate()
+    TechnologiesValidator(os.getenv("TECH_FILE_NAME")).validate()

.github/workflows/validate.yml

@@ -18,6 +18,27 @@ jobs:
       - name: run structure validator
         run: python3 .github/workflows/scripts/structure_validator.py
 
+  validate_schema:
+    runs-on: ubuntu-22.04
+    needs: validate_structure
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+
+      - name: set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install dependencies
+        run: python3 -m pip install jsonschema
+
+      - name: run schema validator
+        run: python3 .github/workflows/scripts/schema_validator.py
+
   validate_categories:
     runs-on: ubuntu-22.04
     needs: validate_structure

.gitignore

@@ -1,4 +1,6 @@
-.idea/
-
-.project
-.settings
+.idea/
+
+.project
+.settings
+
+.sync/

CONTRIBUTING.md

@@ -1,10 +1,10 @@
 # Contributing
 
-WebAppAnalyzer is an [GPLv3 licensed](https://github.com/wappalyzer/wappalyzer/blob/master/LICENSE), open source project written in JavaScript. Anyone is welcome to contribute.
+WebAppAnalyzer is an [GPLv3 licensed](https://github.com/enthec/webappanalyzer/blob/master/LICENSE), open source project written in JavaScript. Anyone is welcome to contribute.
 
 ## Getting started
 
-To get started, see the [README](https://github.com/wappalyzer/wappalyzer/blob/master/README.md).
+To get started, see the [README](https://github.com/enthec/webappanalyzer/blob/master/README.md).
 
 ## Adding a new technology