From 4bd196d8f9bd58bc96c7d350f66c17ece3d54c5a Mon Sep 17 00:00:00 2001
From: Kaustubh BM <kaustubhbm3574@gmail.com>
Date: Tue, 31 Jan 2023 14:45:02 +0530
Subject: [PATCH 1/2] Update Wappalyzer.py

Adding 2 functions:
1) analyze_with_cpe(self,webpage): This function adds the CPE (Common Platform Enumeration) to the output as this is also available in the analyzed data.
2) analyze_with_cpe_and_versions(self,webpage): This function matches the CPE to the corresponding version and outputs that result.
---
 Wappalyzer/Wappalyzer.py | 647 ++++++++++++++++++++-------------------
 1 file changed, 331 insertions(+), 316 deletions(-)

diff --git a/Wappalyzer/Wappalyzer.py b/Wappalyzer/Wappalyzer.py
index 8169048..e1f6ded 100644
--- a/Wappalyzer/Wappalyzer.py
+++ b/Wappalyzer/Wappalyzer.py
@@ -1,270 +1,314 @@
-
-from typing import Callable, Dict, Iterable, List, Any, Mapping, Set
+import aiohttp
+import asyncio
 import json
 import logging
 import pkg_resources
 import re
-import os
-import pathlib
 import requests
+import warnings
 
-from datetime import datetime, timedelta
-from typing import Optional
+from bs4 import BeautifulSoup
+from typing import Union
 
-from Wappalyzer.fingerprint import Fingerprint, Pattern, Technology, Category
-from Wappalyzer.webpage import WebPage, IWebPage
+logger = logging.getLogger(name=__name__)
 
-logger = logging.getLogger(name="python-Wappalyzer")
 
 class WappalyzerError(Exception):
-    # unused for now
     """
     Raised for fatal Wappalyzer errors.
     """
     pass
 
-class Wappalyzer:
+
+class WebPage:
+    """
+    Simple representation of a web page, decoupled
+    from any particular HTTP library's API.
     """
-    Python Wappalyzer driver.
 
-    Consider the following exemples.
-    
-    Here is how you can use the latest technologies file from AliasIO/wappalyzer repository. 
-    
-    .. python::
+    def __init__(self, url, html, headers):
+        """
+        Initialize a new WebPage object.
 
-        from Wappalyzer import Wappalyzer
-        wappalyzer=Wappalyzer.latest(update=True)
-        # Create webpage
-        webpage=WebPage.new_from_url('http://example.com')
-        # analyze
-        results = wappalyzer.analyze_with_categories(webpage)
+        Parameters
+        ----------
 
+        url : str
+            The web page URL.
+        html : str
+            The web page content (HTML)
+        headers : dict
+            The HTTP response headers
+        """
+        self.url = url
+        self.html = html
+        self.headers = headers
 
-    Here is how you can custom request and headers arguments:
-    
-    .. python::
+        try:
+            list(self.headers.keys())
+        except AttributeError:
+            raise ValueError("Headers must be a dictionary-like object")
 
-        import requests
-        from Wappalyzer import Wappalyzer, WebPage
-        wappalyzer = Wappalyzer.latest()
-        webpage = WebPage.new_from_url('http://exemple.com', headers={'User-Agent': 'Custom user agent'})
-        wappalyzer.analyze_with_categories(webpage)
+        self._parse_html()
 
-    """
+    def _parse_html(self):
+        """
+        Parse the HTML with BeautifulSoup to find <script> and <meta> tags.
+        """
+        self.parsed_html = soup = BeautifulSoup(self.html, 'lxml')
+        self.scripts = [script['src'] for script in
+                        soup.findAll('script', src=True)]
+        self.meta = {
+            meta['name'].lower():
+                meta['content'] for meta in soup.findAll(
+                    'meta', attrs=dict(name=True, content=True))
+        }
 
-    def __init__(self, categories:Dict[str, Any], technologies:Dict[str, Any]):
+    @classmethod
+    def new_from_url(cls, url: str, verify: bool = True, timeout: Union[int, float] = 10):
         """
-        Manually initialize a new Wappalyzer instance. 
-        
-        You might want to use the factory method: `latest`
+        Constructs a new WebPage object for the URL,
+        using the `requests` module to fetch the HTML.
+
+        Parameters
+        ----------
 
-        :param categories: Map of category ids to names, as in ``technologies.json``.
-        :param technologies: Map of technology names to technology dicts, as in ``technologies.json``.
+        url : str
+        verify: bool
+        timeout: int, float
+        """
+        response = requests.get(url, verify=verify, timeout=timeout)
+        return cls.new_from_response(response)
+
+    @classmethod
+    async def new_from_url_async(cls, url: str, verify: bool = True, timeout: Union[int, float] = 2.5,
+                                 aiohttp_client_session: aiohttp.ClientSession = None):
+        """
+        Same as new_from_url only Async.
+
+        Constructs a new WebPage object for the URL,
+        using the `aiohttp` module to fetch the HTML.
+
+        Parameters
+        ----------
+
+        url : str
+        verify: bool
+        """
+
+        if not aiohttp_client_session:
+            connector = aiohttp.TCPConnector(ssl=verify)
+            aiohttp_client_session = aiohttp.ClientSession(connector=connector)
+
+        async with aiohttp_client_session.get(url, timeout=timeout) as response:
+            return await cls.new_from_response_async(response)
+
+    @classmethod
+    def new_from_response(cls, response):
         """
-        self.categories: Mapping[str, Category] = {k:Category(**v) for k,v in categories.items()}
-        self.technologies: Mapping[str, Fingerprint] = {k:Fingerprint(name=k, **v) for k,v in technologies.items()}
-        self.detected_technologies: Dict[str, Dict[str, Technology]] = {}
+        Constructs a new WebPage object for the response,
+        using the `BeautifulSoup` module to parse the HTML.
 
-        self._confidence_regexp = re.compile(r"(.+)\\;confidence:(\d+)")
+        Parameters
+        ----------
+
+        response : requests.Response object
+        """
+        return cls(response.url, html=response.text, headers=response.headers)
 
     @classmethod
-    def latest(cls, technologies_file:str=None, update:bool=False) -> 'Wappalyzer':
+    async def new_from_response_async(cls, response):
+        """
+        Constructs a new WebPage object for the response,
+        using the `BeautifulSoup` module to parse the HTML.
+
+        Parameters
+        ----------
+
+        response : aiohttp.ClientResponse¶ object
+        """
+        html = await response.text()
+        return cls(str(response.url), html=html, headers=response.headers)
+
+
+class Wappalyzer:
+    """
+    Python Wappalyzer driver.
+    """
+
+    def __init__(self, categories, technologies):
         """
-        Construct a Wappalyzer instance.
-        
-        Use ``update=True`` to download the very latest file from internet. 
-        Do not update if the file has already been updated in the last 24 hours. 
-        *New in version 0.4.0*
+        Initialize a new Wappalyzer instance.
 
-        Use ``technologies_file=/some/path/technologies.json`` to load a 
-        custom technologies file. 
-        
-        If no arguments is passed, load the default ``data/technologies.json`` file
-        inside the package ressource.
+        Parameters
+        ----------
 
-        :param technologies_file: File path
-        :param update: Download and use the latest ``technologies.json`` file 
-            from `AliasIO/wappalyzer <https://github.com/AliasIO/wappalyzer>`_ repository.  
-        
+        categories : dict
+            Map of category ids to names, as in technologies.json.
+        technologies : dict
+            Map of technology names to technology dicts, as in technologies.json.
         """
-        default=pkg_resources.resource_string(__name__, "data/technologies.json")
-        defaultobj = json.loads(default)
+        self.categories = categories
+        self.technologies = technologies
+        self.confidence_regexp = re.compile(r"(.+)\\;confidence:(\d+)")
 
+        # TODO
+        for name, technology in list(self.technologies.items()):
+            self._prepare_technology(technology)
+
+    @classmethod
+    def latest(cls, technologies_file=None):
+        """
+        Construct a Wappalyzer instance using a technologies db path passed in via
+        technologies_file, or alternatively the default in data/technologies.json
+        """
         if technologies_file:
-            with open(technologies_file, 'r', encoding='utf-8') as fd:
+            with open(technologies_file, 'r') as fd:
                 obj = json.load(fd)
-        elif update:
-            should_update = True
-            _technologies_file: pathlib.Path
-            _files = cls._find_files(['HOME', 'APPDATA',], ['.python-Wappalyzer/technologies.json'])
-            if _files:
-                _technologies_file = pathlib.Path(_files[0])
-                last_modification_time = datetime.fromtimestamp(_technologies_file.stat().st_mtime)
-                if datetime.now() - last_modification_time < timedelta(hours=24):
-                    should_update = False
-
-            # Get the lastest file
-            if should_update:
-                try:
-                    lastest_technologies_file=requests.get('https://raw.githubusercontent.com/AliasIO/wappalyzer/master/src/technologies.json')
-                    obj = lastest_technologies_file.json()
-                    _technologies_file = pathlib.Path(cls._find_files(
-                        ['HOME', 'APPDATA',],
-                        ['.python-Wappalyzer/technologies.json'],
-                        create = True
-                        ).pop())
-                    
-                    if obj != defaultobj:
-                        with _technologies_file.open('w', encoding='utf-8') as tfile:
-                            tfile.write(lastest_technologies_file.text)
-                        logger.info("python-Wappalyzer technologies.json file updated")
-
-                except Exception as err: # Or loads default
-                    logger.error("Could not download latest Wappalyzer technologies.json file because of error : '{}'. Using default. ".format(err))
-                    obj = defaultobj
-            else:
-                logger.debug("python-Wappalyzer technologies.json file not updated because already updated in the last 24h")
-                with _technologies_file.open('r', encoding='utf-8') as tfile:
-                    obj = json.load(tfile)
-
-            logger.info("Using technologies.json file at {}".format(_technologies_file.as_posix()))
         else:
-            obj = defaultobj
+            obj = json.loads(pkg_resources.resource_string(__name__, "data/technologies.json"))
 
-        
         return cls(categories=obj['categories'], technologies=obj['technologies'])
 
-    @staticmethod
-    def _find_files(
-        env_location: List[str],
-        potential_files: List[str],
-        default_content: str = "",
-        create: bool = False,
-    ) -> List[str]:
-        """Find existent files based on folders name and file names.
-        Arguments:
-        - `env_location`: list of environment variable to use as a base path. Exemple: ['HOME', 'XDG_CONFIG_HOME', 'APPDATA', 'PWD']
-        - `potential_files`: list of filenames. Exemple: ['.myapp/conf.ini',]
-        - `default_content`: Write default content if the file does not exist
-        - `create`: Create the file in the first existing env_location with default content if the file does not exist
-        """
-        potential_paths = []
-        existent_files = []
-
-        env_loc_exists = False
-        # build potential_paths of config file
-        for env_var in env_location:
-            if env_var in os.environ:
-                env_loc_exists = True
-                for file_path in potential_files:
-                    potential_paths.append(os.path.join(os.environ[env_var], file_path))
-        if not env_loc_exists and create:
-            raise RuntimeError(f"Cannot find any of the env locations {env_location}. ")
-        # If file exist, add to list
-        for p in potential_paths:
-            if os.path.isfile(p):
-                existent_files.append(p)
-        # If no file foud and create=True, init new file
-        if len(existent_files) == 0 and create:
-            os.makedirs(os.path.dirname(potential_paths[0]), exist_ok=True)
-            with open(potential_paths[0], "w", encoding='utf-8') as config_file:
-                config_file.write(default_content)
-            existent_files.append(potential_paths[0])
-        return existent_files
-
-    def _has_technology(self, tech_fingerprint: Fingerprint, webpage: IWebPage) -> bool:
+    def _prepare_technology(self, technology):
+        """
+        Normalize technology data, preparing it for the detection phase.
+        """
+        # Ensure these keys' values are lists
+        for key in ['url', 'html', 'scripts', 'implies']:
+            try:
+                value = technology[key]
+            except KeyError:
+                technology[key] = []
+            else:
+                if not isinstance(value, list):
+                    technology[key] = [value]
+
+        # Ensure these keys exist
+        for key in ['headers', 'meta']:
+            try:
+                value = technology[key]
+            except KeyError:
+                technology[key] = {}
+
+        # Ensure the 'meta' key is a dict
+        obj = technology['meta']
+        if not isinstance(obj, dict):
+            technology['meta'] = {'generator': obj}
+
+        # Ensure keys are lowercase
+        for key in ['headers', 'meta']:
+            obj = technology[key]
+            technology[key] = {k.lower(): v for k, v in list(obj.items())}
+
+        # Prepare regular expression patterns
+        for key in ['url', 'html', 'scripts']:
+            technology[key] = [self._prepare_pattern(pattern) for pattern in technology[key]]
+
+        for key in ['headers', 'meta']:
+            obj = technology[key]
+            for name, pattern in list(obj.items()):
+                obj[name] = self._prepare_pattern(obj[name])
+
+    def _prepare_pattern(self, pattern):
+        """
+        Strip out key:value pairs from the pattern and compile the regular
+        expression.
+        """
+        attrs = {}
+        pattern = pattern.split('\\;')
+        for index, expression in enumerate(pattern):
+            if index == 0:
+                attrs['string'] = expression
+                try:
+                    attrs['regex'] = re.compile(expression, re.I)
+                except re.error as err:
+                    warnings.warn(
+                        "Caught '{error}' compiling regex: {regex}".format(
+                            error=err, regex=pattern)
+                    )
+                    # regex that never matches:
+                    # http://stackoverflow.com/a/1845097/413622
+                    attrs['regex'] = re.compile(r'(?!x)x')
+            else:
+                attr = expression.split(':')
+                if len(attr) > 1:
+                    key = attr.pop(0)
+                    attrs[str(key)] = ':'.join(attr)
+        return attrs
+
+    def _has_technology(self, technology, webpage):
         """
         Determine whether the web page matches the technology signature.
         """
+        app = technology
 
-        has_tech = False
+        has_app = False
         # Search the easiest things first and save the full-text search of the
         # HTML for last
 
-        # analyze url patterns
-        for pattern in tech_fingerprint.url:
-            if pattern.regex.search(webpage.url):
-                self._set_detected_app(webpage.url, tech_fingerprint, 'url', pattern, value=webpage.url)
-        # analyze headers patterns
-        for name, patterns in list(tech_fingerprint.headers.items()):
+        for pattern in app['url']:
+            if pattern['regex'].search(webpage.url):
+                self._set_detected_app(app, 'url', pattern, webpage.url)
+
+        for name, pattern in list(app['headers'].items()):
             if name in webpage.headers:
                 content = webpage.headers[name]
-                for pattern in patterns:
-                    if pattern.regex.search(content):
-                        self._set_detected_app(webpage.url, tech_fingerprint, 'headers', pattern, value=content, key=name)
-                        has_tech = True
-        # analyze scripts patterns
-        for pattern in tech_fingerprint.scripts:
+                if pattern['regex'].search(content):
+                    self._set_detected_app(app, 'headers', pattern, content, name)
+                    has_app = True
+
+        for pattern in technology['scripts']:
             for script in webpage.scripts:
-                if pattern.regex.search(script):
-                    self._set_detected_app(webpage.url, tech_fingerprint, 'scripts', pattern, value=script)
-                    has_tech = True
-        # analyze meta patterns
-        for name, patterns in list(tech_fingerprint.meta.items()):
+                if pattern['regex'].search(script):
+                    self._set_detected_app(app, 'scripts', pattern, script)
+                    has_app = True
+
+        for name, pattern in list(technology['meta'].items()):
             if name in webpage.meta:
                 content = webpage.meta[name]
-                for pattern in patterns:
-                    if pattern.regex.search(content):
-                        self._set_detected_app(webpage.url, tech_fingerprint, 'meta', pattern, value=content, key=name)
-                        has_tech = True
-        # analyze html patterns
-        for pattern in tech_fingerprint.html:
-            if pattern.regex.search(webpage.html):
-                self._set_detected_app(webpage.url, tech_fingerprint, 'html', pattern, value=webpage.html)
-                has_tech = True
-        # analyze dom patterns
-        # css selector, list of css selectors, or dict from css selector to dict with some of keys:
-        #           - "exists": "": only check if the selector matches somthing, equivalent to the list form. 
-        #           - "text": "regex": check if the .innerText property of the element that matches the css selector matches the regex (with version extraction).
-        #           - "attributes": {dict from attr name to regex}: check if the attribute value of the element that matches the css selector matches the regex (with version extraction).
-        for selector in tech_fingerprint.dom:
-            for item in webpage.select(selector.selector):
-                if selector.exists:
-                    self._set_detected_app(webpage.url, tech_fingerprint, 'dom', Pattern(string=selector.selector), value='')
-                    has_tech = True
-                if selector.text:
-                    for pattern in selector.text:
-                        if pattern.regex.search(item.inner_html):
-                            self._set_detected_app(webpage.url, tech_fingerprint, 'dom', pattern, value=item.inner_html)
-                            has_tech = True
-                if selector.attributes:
-                    for attrname, patterns in list(selector.attributes.items()):
-                        _content = item.attributes.get(attrname)
-                        if _content:
-                            for pattern in patterns:
-                                if pattern.regex.search(_content):
-                                    self._set_detected_app(webpage.url, tech_fingerprint, 'dom', pattern, value=_content)
-                                    has_tech = True
-        return has_tech
-
-    def _set_detected_app(self, url:str,
-                                tech_fingerprint: Fingerprint, 
-                                app_type:str, 
-                                pattern: Pattern, 
-                                value:str, 
-                                key='') -> None:
-        """
-        Store detected technology to the detected_technologies dict.
-        """
-        # Lookup Technology object in the cache
-        if url not in self.detected_technologies:
-            self.detected_technologies[url] = {}
-        if tech_fingerprint.name not in self.detected_technologies[url]:
-            self.detected_technologies[url][tech_fingerprint.name] = Technology(tech_fingerprint.name)
-        detected_tech = self.detected_technologies[url][tech_fingerprint.name]
+                if pattern['regex'].search(content):
+                    self._set_detected_app(app, 'meta', pattern, content, name)
+                    has_app = True
+
+        for pattern in app['html']:
+            if pattern['regex'].search(webpage.html):
+                self._set_detected_app(app, 'html', pattern, webpage.html)
+                has_app = True
+
+        # Set total confidence
+        if has_app:
+            total = 0
+            for index in app['confidence']:
+                total += app['confidence'][index]
+            app['confidenceTotal'] = total
+
+        return has_app
+
+    def _set_detected_app(self, app, app_type, pattern, value, key=''):
+        """
+        Store detected app.
+        """
+        app['detected'] = True
 
         # Set confidence level
-        if key != '': key += ' '
-        match_name = app_type + ' ' + key + pattern.string
-        
-        detected_tech.confidence[match_name] = pattern.confidence
+        if key != '':
+            key += ' '
+        if 'confidence' not in app:
+            app['confidence'] = {}
+        if 'confidence' not in pattern:
+            pattern['confidence'] = 100
+        else:
+            # Convert to int for easy adding later
+            pattern['confidence'] = int(pattern['confidence'])
+        app['confidence'][app_type + ' ' + key + pattern['string']] = pattern['confidence']
 
         # Dectect version number
-        if pattern.version:
-            allmatches = re.findall(pattern.regex, value)
+        if 'version' in pattern:
+            allmatches = re.findall(pattern['regex'], value)
             for i, matches in enumerate(allmatches):
-                version = pattern.version
+                version = pattern['version']
+
                 # Check for a string to avoid enumerating the string
                 if isinstance(matches, str):
                     matches = [(matches)]
@@ -274,29 +318,34 @@ def _set_detected_app(self, url:str,
                     if ternary and len(ternary.groups()) == 2 and ternary.group(1) is not None and ternary.group(2) is not None:
                         version = version.replace(ternary.group(0), ternary.group(1) if match != ''
                                                   else ternary.group(2))
+
                     # Replace back references
                     version = version.replace('\\' + str(index + 1), match)
-                if version != '' and version not in detected_tech.versions:
-                    detected_tech.versions.append(version)
-            self._sort_app_version(detected_tech)
-
-    def _sort_app_version(self, detected_tech: Technology) -> None:
+                if version != '':
+                    if 'versions' not in app:
+                        app['versions'] = [version]
+                    elif version not in app['versions']:
+                        app['versions'].append(version)
+            self._set_app_version(app)
+
+    def _set_app_version(self, app):
         """
-        Sort version number (find the longest version number that *is supposed to* contains all shorter detected version numbers).
+        Resolve version number (find the longest version number that contains all shorter detected version numbers).
         """
-        if len(detected_tech.versions) >= 1:
+        if 'versions' not in app:
             return
-        detected_tech.versions = sorted(detected_tech.versions, key=self._cmp_to_key(self._sort_app_versions))
 
-    def _get_implied_technologies(self, detected_technologies:Iterable[str]) -> Iterable[str]:
+        app['versions'] = sorted(app['versions'], key=self._cmp_to_key(self._sort_app_versions))
+
+    def _get_implied_technologies(self, detected_technologies):
         """
         Get the set of technologies implied by `detected_technologies`.
         """
-        def __get_implied_technologies(technologies:Iterable[str]) -> Iterable[str] :
+        def __get_implied_technologies(technologies):
             _implied_technologies = set()
             for tech in technologies:
                 try:
-                    for implie in self.technologies[tech].implies:
+                    for implie in self.technologies[tech]['implies']:
                         # If we have no doubts just add technology
                         if 'confidence' not in implie:
                             _implied_technologies.add(implie)
@@ -306,7 +355,7 @@ def __get_implied_technologies(technologies:Iterable[str]) -> Iterable[str] :
                             try:
                                 # Use more strict regexp (cause we have already checked the entry of "confidence")
                                 # Also, better way to compile regexp one time, instead of every time
-                                app_name, confidence = self._confidence_regexp.search(implie).groups() # type: ignore
+                                app_name, confidence = self.confidence_regexp.search(implie).groups()
                                 if int(confidence) >= 50:
                                     _implied_technologies.add(app_name)
                             except (ValueError, AttributeError):
@@ -316,7 +365,7 @@ def __get_implied_technologies(technologies:Iterable[str]) -> Iterable[str] :
             return _implied_technologies
 
         implied_technologies = __get_implied_technologies(detected_technologies)
-        all_implied_technologies : Set[str] = set()
+        all_implied_technologies = set()
 
         # Descend recursively until we've found all implied technologies
         while not all_implied_technologies.issuperset(implied_technologies):
@@ -325,46 +374,37 @@ def __get_implied_technologies(technologies:Iterable[str]) -> Iterable[str] :
 
         return all_implied_technologies
 
-    def get_categories(self, tech_name:str) -> List[str]:
+    def get_categories(self, tech_name):
         """
         Returns a list of the categories for an technology name.
-
-        :param tech_name: Tech name
         """
-        cat_nums = self.technologies[tech_name].cats if tech_name in self.technologies else []
-        cat_names = [self.categories[str(cat_num)].name
-                     for cat_num in cat_nums if str(cat_num) in self.categories]
+        cat_nums = self.technologies.get(tech_name, {}).get("cats", [])
+        cat_names = [self.categories.get(str(cat_num), "").get("name", "")
+                     for cat_num in cat_nums]
+
         return cat_names
 
-    def get_versions(self, url:str, app_name:str) -> List[str]:
+    def get_versions(self, app_name):
         """
         Retuns a list of the discovered versions for an app name.
+        """
+        return [] if 'versions' not in self.technologies[app_name] else self.technologies[app_name]['versions']
 
-        :param url: URL of the webpage
-        :param app_name: App name
+    def get_cpe(self, app_name):
         """
-        try:
-            return self.detected_technologies[url][app_name].versions
-        except KeyError:
-            return []
+        Retuns a list of the discovered cpes for an app name.
+        """
+        return [] if 'cpe' not in self.technologies[app_name] else self.technologies[app_name]['cpe']
 
-    def get_confidence(self, url:str, app_name:str) -> Optional[int]:
+    def get_confidence(self, app_name):
         """
         Returns the total confidence for an app name.
-
-        :param url: URL of the webpage
-        :param app_name: App name
         """
-        try:
-            return self.detected_technologies[url][app_name].confidenceTotal
-        except KeyError:
-            return None
+        return [] if 'confidenceTotal' not in self.technologies[app_name] else self.technologies[app_name]['confidenceTotal']
 
-    def analyze(self, webpage:IWebPage) -> Set[str]:
+    def analyze(self, webpage):
         """
-        Return a set of technology that can be detected on the web page.
-
-        :param webpage: The Webpage to analyze
+        Return a list of technologylications that can be detected on the web page.
         """
         detected_technologies = set()
 
@@ -372,37 +412,62 @@ def analyze(self, webpage:IWebPage) -> Set[str]:
             if self._has_technology(technology, webpage):
                 detected_technologies.add(tech_name)
 
-        detected_technologies.update(self._get_implied_technologies(detected_technologies))
-
+        detected_technologies |= self._get_implied_technologies(detected_technologies)
         return detected_technologies
 
-    def analyze_with_versions(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]:
+    def analyze_with_versions(self, webpage):
         """
-        Return a dict of applications and versions that can be detected on the web page.
-
-        :param webpage: The Webpage to analyze
+        Return a list of applications and versions that can be detected on the web page.
         """
         detected_apps = self.analyze(webpage)
         versioned_apps = {}
 
         for app_name in detected_apps:
-            versions = self.get_versions(webpage.url, app_name)
+            versions = self.get_versions(app_name)
             versioned_apps[app_name] = {"versions": versions}
 
         return versioned_apps
 
-    def analyze_with_categories(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]:
+    def analyze_with_cpe(self, webpage):
         """
-        Return a dict of technologies and categories that can be detected on the web page.
+        Return a list of applications and versions that can be detected on the web page.
+        """
+        detected_apps = self.analyze(webpage)
+        cpe_detected_apps = {}
 
-        :param webpage: The Webpage to analyze
+        for app_name in detected_apps:
+            cpes = self.get_cpe(app_name)
+            cpe_detected_apps[app_name] = {"cpes": cpes}
 
-        >>> wappalyzer.analyze_with_categories(webpage)
-        {'Amazon ECS': {'categories': ['IaaS']},
-        'Amazon Web Services': {'categories': ['PaaS']},
-        'Azure CDN': {'categories': ['CDN']},
-        'Docker': {'categories': ['Containers']}}
+        return cpe_detected_apps
 
+    def analyze_with_cpe_and_version(self, webpage):
+        """
+        Return a list of applications and versions that can be detected on the web page.
+        """
+        detected_apps = self.analyze(webpage)
+        cpe_detected_apps = {}
+
+        for app_name in detected_apps:
+            cpe = self.get_cpe(app_name)
+            versions = self.get_versions(app_name)
+            if cpe == []:
+                cpe_detected_apps[app_name] = {"Versions": versions}
+            else:
+                cpe = cpe.replace("/","2.3:") 
+                full_cpes = []
+                for version in versions:
+                    full_cpes.append(cpe + ":" + version)
+
+                cpe_detected_apps[app_name] = {"CPEs": full_cpes}
+
+
+        return cpe_detected_apps
+
+
+    def analyze_with_categories(self, webpage):
+        """
+        Return a list of technologies and categories that can be detected on the web page.
         """
         detected_technologies = self.analyze(webpage)
         categorised_technologies = {}
@@ -413,22 +478,7 @@ def analyze_with_categories(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]
 
         return categorised_technologies
 
-    def analyze_with_versions_and_categories(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]:
-        """
-        Return a dict of applications and versions and categories that can be detected on the web page.
-
-        :param webpage: The Webpage to analyze
-
-        >>> wappalyzer.analyze_with_versions_and_categories(webpage)
-        {'Font Awesome': {'categories': ['Font scripts'], 'versions': ['5.4.2']},
-        'Google Font API': {'categories': ['Font scripts'], 'versions': []},
-        'MySQL': {'categories': ['Databases'], 'versions': []},
-        'Nginx': {'categories': ['Web servers', 'Reverse proxies'], 'versions': []},
-        'PHP': {'categories': ['Programming languages'], 'versions': ['5.6.40']},
-        'WordPress': {'categories': ['CMS', 'Blogs'], 'versions': ['5.4.2']},
-        'Yoast SEO': {'categories': ['SEO'], 'versions': ['14.6.1']}}
-
-        """
+    def analyze_with_versions_and_categories(self, webpage):
         versioned_apps = self.analyze_with_versions(webpage)
         versioned_and_categorised_apps = versioned_apps
 
@@ -438,10 +488,10 @@ def analyze_with_versions_and_categories(self, webpage:IWebPage) -> Dict[str, Di
 
         return versioned_and_categorised_apps
 
-    def _sort_app_versions(self, version_a: str, version_b: str) -> int:
+    def _sort_app_versions(self, version_a, version_b):
         return len(version_a) - len(version_b)
 
-    def _cmp_to_key(self, mycmp: Callable[..., Any]):
+    def _cmp_to_key(self, mycmp):
         """
         Convert a cmp= function into a key= function
         """
@@ -470,38 +520,3 @@ def __ne__(self, other):
                 return mycmp(self.obj, other.obj) != 0
 
         return CmpToKey
-
-def analyze(url:str, 
-            update:bool=False, 
-            useragent:str=None,
-            timeout:int=10,
-            verify:bool=True) -> Dict[str, Dict[str, Any]]:
-    """
-    Quick utility method to analyze a website with minimal configurable options. 
-
-    :See: `WebPage` and `Wappalyzer`. 
-
-    :Parameters:
-        - `url`: URL
-        - `update`: Update the technologies file from the internet
-        - `useragent`: Request user agent
-        - `timeout`: Request timeout
-        - `verify`: SSL cert verify
-    
-    :Return: 
-        `dict`. Just as `Wappalyzer.analyze_with_versions_and_categories`. 
-    :Note: More information might be added to the returned values in the future
-    """
-    # Create Wappalyzer
-    wappalyzer=Wappalyzer.latest(update=update)
-    # Create WebPage
-    headers={}
-    if useragent:
-        headers['User-Agent'] = useragent
-    webpage=WebPage.new_from_url(url, 
-        headers=headers, 
-        timeout=timeout, 
-        verify=verify)
-    # Analyze
-    results = wappalyzer.analyze_with_versions_and_categories(webpage)
-    return results

From a4d6c8c896721bdba2fe72717fb288055d80532a Mon Sep 17 00:00:00 2001
From: Kaustubh BM <kaustubhbm3574@gmail.com>
Date: Tue, 31 Jan 2023 15:13:44 +0530
Subject: [PATCH 2/2] Update Wappalyzer.py

Adding 2 functions to allow the user to get the CPE (Common Platform Enumeration) of the application discovered.
This makes it easier for the user to co-relate the results with various other databases.
---
 Wappalyzer/Wappalyzer.py | 621 +++++++++++++++++++--------------------
 1 file changed, 310 insertions(+), 311 deletions(-)

diff --git a/Wappalyzer/Wappalyzer.py b/Wappalyzer/Wappalyzer.py
index e1f6ded..8f5e077 100644
--- a/Wappalyzer/Wappalyzer.py
+++ b/Wappalyzer/Wappalyzer.py
@@ -1,314 +1,260 @@
-import aiohttp
-import asyncio
+from typing import Callable, Dict, Iterable, List, Any, Mapping, Set
 import json
 import logging
 import pkg_resources
 import re
+import os
+import pathlib
 import requests
-import warnings
 
-from bs4 import BeautifulSoup
-from typing import Union
+from datetime import datetime, timedelta
+from typing import Optional
 
-logger = logging.getLogger(name=__name__)
+from Wappalyzer.fingerprint import Fingerprint, Pattern, Technology, Category
+from Wappalyzer.webpage import WebPage, IWebPage
 
+logger = logging.getLogger(name="python-Wappalyzer")
 
 class WappalyzerError(Exception):
+    # unused for now
     """
     Raised for fatal Wappalyzer errors.
     """
     pass
 
-
-class WebPage:
-    """
-    Simple representation of a web page, decoupled
-    from any particular HTTP library's API.
-    """
-
-    def __init__(self, url, html, headers):
-        """
-        Initialize a new WebPage object.
-
-        Parameters
-        ----------
-
-        url : str
-            The web page URL.
-        html : str
-            The web page content (HTML)
-        headers : dict
-            The HTTP response headers
-        """
-        self.url = url
-        self.html = html
-        self.headers = headers
-
-        try:
-            list(self.headers.keys())
-        except AttributeError:
-            raise ValueError("Headers must be a dictionary-like object")
-
-        self._parse_html()
-
-    def _parse_html(self):
-        """
-        Parse the HTML with BeautifulSoup to find <script> and <meta> tags.
-        """
-        self.parsed_html = soup = BeautifulSoup(self.html, 'lxml')
-        self.scripts = [script['src'] for script in
-                        soup.findAll('script', src=True)]
-        self.meta = {
-            meta['name'].lower():
-                meta['content'] for meta in soup.findAll(
-                    'meta', attrs=dict(name=True, content=True))
-        }
-
-    @classmethod
-    def new_from_url(cls, url: str, verify: bool = True, timeout: Union[int, float] = 10):
-        """
-        Constructs a new WebPage object for the URL,
-        using the `requests` module to fetch the HTML.
-
-        Parameters
-        ----------
-
-        url : str
-        verify: bool
-        timeout: int, float
-        """
-        response = requests.get(url, verify=verify, timeout=timeout)
-        return cls.new_from_response(response)
-
-    @classmethod
-    async def new_from_url_async(cls, url: str, verify: bool = True, timeout: Union[int, float] = 2.5,
-                                 aiohttp_client_session: aiohttp.ClientSession = None):
-        """
-        Same as new_from_url only Async.
-
-        Constructs a new WebPage object for the URL,
-        using the `aiohttp` module to fetch the HTML.
-
-        Parameters
-        ----------
-
-        url : str
-        verify: bool
-        """
-
-        if not aiohttp_client_session:
-            connector = aiohttp.TCPConnector(ssl=verify)
-            aiohttp_client_session = aiohttp.ClientSession(connector=connector)
-
-        async with aiohttp_client_session.get(url, timeout=timeout) as response:
-            return await cls.new_from_response_async(response)
-
-    @classmethod
-    def new_from_response(cls, response):
-        """
-        Constructs a new WebPage object for the response,
-        using the `BeautifulSoup` module to parse the HTML.
-
-        Parameters
-        ----------
-
-        response : requests.Response object
-        """
-        return cls(response.url, html=response.text, headers=response.headers)
-
-    @classmethod
-    async def new_from_response_async(cls, response):
-        """
-        Constructs a new WebPage object for the response,
-        using the `BeautifulSoup` module to parse the HTML.
-
-        Parameters
-        ----------
-
-        response : aiohttp.ClientResponse¶ object
-        """
-        html = await response.text()
-        return cls(str(response.url), html=html, headers=response.headers)
-
-
 class Wappalyzer:
     """
     Python Wappalyzer driver.
+    Consider the following exemples.
+    
+    Here is how you can use the latest technologies file from AliasIO/wappalyzer repository. 
+    
+    .. python::
+        from Wappalyzer import Wappalyzer
+        wappalyzer=Wappalyzer.latest(update=True)
+        # Create webpage
+        webpage=WebPage.new_from_url('http://example.com')
+        # analyze
+        results = wappalyzer.analyze_with_categories(webpage)
+    Here is how you can custom request and headers arguments:
+    
+    .. python::
+        import requests
+        from Wappalyzer import Wappalyzer, WebPage
+        wappalyzer = Wappalyzer.latest()
+        webpage = WebPage.new_from_url('http://exemple.com', headers={'User-Agent': 'Custom user agent'})
+        wappalyzer.analyze_with_categories(webpage)
     """
 
-    def __init__(self, categories, technologies):
+    def __init__(self, categories:Dict[str, Any], technologies:Dict[str, Any]):
         """
-        Initialize a new Wappalyzer instance.
-
-        Parameters
-        ----------
-
-        categories : dict
-            Map of category ids to names, as in technologies.json.
-        technologies : dict
-            Map of technology names to technology dicts, as in technologies.json.
+        Manually initialize a new Wappalyzer instance. 
+        
+        You might want to use the factory method: `latest`
+        :param categories: Map of category ids to names, as in ``technologies.json``.
+        :param technologies: Map of technology names to technology dicts, as in ``technologies.json``.
         """
-        self.categories = categories
-        self.technologies = technologies
-        self.confidence_regexp = re.compile(r"(.+)\\;confidence:(\d+)")
+        self.categories: Mapping[str, Category] = {k:Category(**v) for k,v in categories.items()}
+        self.technologies: Mapping[str, Fingerprint] = {k:Fingerprint(name=k, **v) for k,v in technologies.items()}
+        self.detected_technologies: Dict[str, Dict[str, Technology]] = {}
 
-        # TODO
-        for name, technology in list(self.technologies.items()):
-            self._prepare_technology(technology)
+        self._confidence_regexp = re.compile(r"(.+)\\;confidence:(\d+)")
 
     @classmethod
-    def latest(cls, technologies_file=None):
-        """
-        Construct a Wappalyzer instance using a technologies db path passed in via
-        technologies_file, or alternatively the default in data/technologies.json
-        """
+    def latest(cls, technologies_file:str=None, update:bool=False) -> 'Wappalyzer':
+        """
+        Construct a Wappalyzer instance.
+        
+        Use ``update=True`` to download the very latest file from internet. 
+        Do not update if the file has already been updated in the last 24 hours. 
+        *New in version 0.4.0*
+        Use ``technologies_file=/some/path/technologies.json`` to load a 
+        custom technologies file. 
+        
+        If no arguments is passed, load the default ``data/technologies.json`` file
+        inside the package ressource.
+        :param technologies_file: File path
+        :param update: Download and use the latest ``technologies.json`` file 
+            from `AliasIO/wappalyzer <https://github.com/AliasIO/wappalyzer>`_ repository.  
+        
+        """
+        default=pkg_resources.resource_string(__name__, "data/technologies.json")
+        defaultobj = json.loads(default)
+
         if technologies_file:
-            with open(technologies_file, 'r') as fd:
+            with open(technologies_file, 'r', encoding='utf-8') as fd:
                 obj = json.load(fd)
+        elif update:
+            should_update = True
+            _technologies_file: pathlib.Path
+            _files = cls._find_files(['HOME', 'APPDATA',], ['.python-Wappalyzer/technologies.json'])
+            if _files:
+                _technologies_file = pathlib.Path(_files[0])
+                last_modification_time = datetime.fromtimestamp(_technologies_file.stat().st_mtime)
+                if datetime.now() - last_modification_time < timedelta(hours=24):
+                    should_update = False
+
+            # Get the lastest file
+            if should_update:
+                try:
+                    lastest_technologies_file=requests.get('https://raw.githubusercontent.com/AliasIO/wappalyzer/master/src/technologies.json')
+                    obj = lastest_technologies_file.json()
+                    _technologies_file = pathlib.Path(cls._find_files(
+                        ['HOME', 'APPDATA',],
+                        ['.python-Wappalyzer/technologies.json'],
+                        create = True
+                        ).pop())
+                    
+                    if obj != defaultobj:
+                        with _technologies_file.open('w', encoding='utf-8') as tfile:
+                            tfile.write(lastest_technologies_file.text)
+                        logger.info("python-Wappalyzer technologies.json file updated")
+
+                except Exception as err: # Or loads default
+                    logger.error("Could not download latest Wappalyzer technologies.json file because of error : '{}'. Using default. ".format(err))
+                    obj = defaultobj
+            else:
+                logger.debug("python-Wappalyzer technologies.json file not updated because already updated in the last 24h")
+                with _technologies_file.open('r', encoding='utf-8') as tfile:
+                    obj = json.load(tfile)
+
+            logger.info("Using technologies.json file at {}".format(_technologies_file.as_posix()))
         else:
-            obj = json.loads(pkg_resources.resource_string(__name__, "data/technologies.json"))
+            obj = defaultobj
 
+        
         return cls(categories=obj['categories'], technologies=obj['technologies'])
 
-    def _prepare_technology(self, technology):
-        """
-        Normalize technology data, preparing it for the detection phase.
-        """
-        # Ensure these keys' values are lists
-        for key in ['url', 'html', 'scripts', 'implies']:
-            try:
-                value = technology[key]
-            except KeyError:
-                technology[key] = []
-            else:
-                if not isinstance(value, list):
-                    technology[key] = [value]
-
-        # Ensure these keys exist
-        for key in ['headers', 'meta']:
-            try:
-                value = technology[key]
-            except KeyError:
-                technology[key] = {}
-
-        # Ensure the 'meta' key is a dict
-        obj = technology['meta']
-        if not isinstance(obj, dict):
-            technology['meta'] = {'generator': obj}
-
-        # Ensure keys are lowercase
-        for key in ['headers', 'meta']:
-            obj = technology[key]
-            technology[key] = {k.lower(): v for k, v in list(obj.items())}
-
-        # Prepare regular expression patterns
-        for key in ['url', 'html', 'scripts']:
-            technology[key] = [self._prepare_pattern(pattern) for pattern in technology[key]]
-
-        for key in ['headers', 'meta']:
-            obj = technology[key]
-            for name, pattern in list(obj.items()):
-                obj[name] = self._prepare_pattern(obj[name])
-
-    def _prepare_pattern(self, pattern):
-        """
-        Strip out key:value pairs from the pattern and compile the regular
-        expression.
-        """
-        attrs = {}
-        pattern = pattern.split('\\;')
-        for index, expression in enumerate(pattern):
-            if index == 0:
-                attrs['string'] = expression
-                try:
-                    attrs['regex'] = re.compile(expression, re.I)
-                except re.error as err:
-                    warnings.warn(
-                        "Caught '{error}' compiling regex: {regex}".format(
-                            error=err, regex=pattern)
-                    )
-                    # regex that never matches:
-                    # http://stackoverflow.com/a/1845097/413622
-                    attrs['regex'] = re.compile(r'(?!x)x')
-            else:
-                attr = expression.split(':')
-                if len(attr) > 1:
-                    key = attr.pop(0)
-                    attrs[str(key)] = ':'.join(attr)
-        return attrs
-
-    def _has_technology(self, technology, webpage):
+    @staticmethod
+    def _find_files(
+        env_location: List[str],
+        potential_files: List[str],
+        default_content: str = "",
+        create: bool = False,
+    ) -> List[str]:
+        """Find existent files based on folders name and file names.
+        Arguments:
+        - `env_location`: list of environment variable to use as a base path. Exemple: ['HOME', 'XDG_CONFIG_HOME', 'APPDATA', 'PWD']
+        - `potential_files`: list of filenames. Exemple: ['.myapp/conf.ini',]
+        - `default_content`: Write default content if the file does not exist
+        - `create`: Create the file in the first existing env_location with default content if the file does not exist
+        """
+        potential_paths = []
+        existent_files = []
+
+        env_loc_exists = False
+        # build potential_paths of config file
+        for env_var in env_location:
+            if env_var in os.environ:
+                env_loc_exists = True
+                for file_path in potential_files:
+                    potential_paths.append(os.path.join(os.environ[env_var], file_path))
+        if not env_loc_exists and create:
+            raise RuntimeError(f"Cannot find any of the env locations {env_location}. ")
+        # If file exist, add to list
+        for p in potential_paths:
+            if os.path.isfile(p):
+                existent_files.append(p)
+        # If no file foud and create=True, init new file
+        if len(existent_files) == 0 and create:
+            os.makedirs(os.path.dirname(potential_paths[0]), exist_ok=True)
+            with open(potential_paths[0], "w", encoding='utf-8') as config_file:
+                config_file.write(default_content)
+            existent_files.append(potential_paths[0])
+        return existent_files
+
+    def _has_technology(self, tech_fingerprint: Fingerprint, webpage: IWebPage) -> bool:
         """
         Determine whether the web page matches the technology signature.
         """
-        app = technology
 
-        has_app = False
+        has_tech = False
         # Search the easiest things first and save the full-text search of the
         # HTML for last
 
-        for pattern in app['url']:
-            if pattern['regex'].search(webpage.url):
-                self._set_detected_app(app, 'url', pattern, webpage.url)
-
-        for name, pattern in list(app['headers'].items()):
+        # analyze url patterns
+        for pattern in tech_fingerprint.url:
+            if pattern.regex.search(webpage.url):
+                self._set_detected_app(webpage.url, tech_fingerprint, 'url', pattern, value=webpage.url)
+        # analyze headers patterns
+        for name, patterns in list(tech_fingerprint.headers.items()):
             if name in webpage.headers:
                 content = webpage.headers[name]
-                if pattern['regex'].search(content):
-                    self._set_detected_app(app, 'headers', pattern, content, name)
-                    has_app = True
-
-        for pattern in technology['scripts']:
+                for pattern in patterns:
+                    if pattern.regex.search(content):
+                        self._set_detected_app(webpage.url, tech_fingerprint, 'headers', pattern, value=content, key=name)
+                        has_tech = True
+        # analyze scripts patterns
+        for pattern in tech_fingerprint.scripts:
             for script in webpage.scripts:
-                if pattern['regex'].search(script):
-                    self._set_detected_app(app, 'scripts', pattern, script)
-                    has_app = True
-
-        for name, pattern in list(technology['meta'].items()):
+                if pattern.regex.search(script):
+                    self._set_detected_app(webpage.url, tech_fingerprint, 'scripts', pattern, value=script)
+                    has_tech = True
+        # analyze meta patterns
+        for name, patterns in list(tech_fingerprint.meta.items()):
             if name in webpage.meta:
                 content = webpage.meta[name]
-                if pattern['regex'].search(content):
-                    self._set_detected_app(app, 'meta', pattern, content, name)
-                    has_app = True
-
-        for pattern in app['html']:
-            if pattern['regex'].search(webpage.html):
-                self._set_detected_app(app, 'html', pattern, webpage.html)
-                has_app = True
-
-        # Set total confidence
-        if has_app:
-            total = 0
-            for index in app['confidence']:
-                total += app['confidence'][index]
-            app['confidenceTotal'] = total
-
-        return has_app
-
-    def _set_detected_app(self, app, app_type, pattern, value, key=''):
-        """
-        Store detected app.
-        """
-        app['detected'] = True
+                for pattern in patterns:
+                    if pattern.regex.search(content):
+                        self._set_detected_app(webpage.url, tech_fingerprint, 'meta', pattern, value=content, key=name)
+                        has_tech = True
+        # analyze html patterns
+        for pattern in tech_fingerprint.html:
+            if pattern.regex.search(webpage.html):
+                self._set_detected_app(webpage.url, tech_fingerprint, 'html', pattern, value=webpage.html)
+                has_tech = True
+        # analyze dom patterns
+        # css selector, list of css selectors, or dict from css selector to dict with some of keys:
+        #           - "exists": "": only check if the selector matches somthing, equivalent to the list form. 
+        #           - "text": "regex": check if the .innerText property of the element that matches the css selector matches the regex (with version extraction).
+        #           - "attributes": {dict from attr name to regex}: check if the attribute value of the element that matches the css selector matches the regex (with version extraction).
+        for selector in tech_fingerprint.dom:
+            for item in webpage.select(selector.selector):
+                if selector.exists:
+                    self._set_detected_app(webpage.url, tech_fingerprint, 'dom', Pattern(string=selector.selector), value='')
+                    has_tech = True
+                if selector.text:
+                    for pattern in selector.text:
+                        if pattern.regex.search(item.inner_html):
+                            self._set_detected_app(webpage.url, tech_fingerprint, 'dom', pattern, value=item.inner_html)
+                            has_tech = True
+                if selector.attributes:
+                    for attrname, patterns in list(selector.attributes.items()):
+                        _content = item.attributes.get(attrname)
+                        if _content:
+                            for pattern in patterns:
+                                if pattern.regex.search(_content):
+                                    self._set_detected_app(webpage.url, tech_fingerprint, 'dom', pattern, value=_content)
+                                    has_tech = True
+        return has_tech
+
+    def _set_detected_app(self, url:str,
+                                tech_fingerprint: Fingerprint, 
+                                app_type:str, 
+                                pattern: Pattern, 
+                                value:str, 
+                                key='') -> None:
+        """
+        Store detected technology to the detected_technologies dict.
+        """
+        # Lookup Technology object in the cache
+        if url not in self.detected_technologies:
+            self.detected_technologies[url] = {}
+        if tech_fingerprint.name not in self.detected_technologies[url]:
+            self.detected_technologies[url][tech_fingerprint.name] = Technology(tech_fingerprint.name)
+        detected_tech = self.detected_technologies[url][tech_fingerprint.name]
 
         # Set confidence level
-        if key != '':
-            key += ' '
-        if 'confidence' not in app:
-            app['confidence'] = {}
-        if 'confidence' not in pattern:
-            pattern['confidence'] = 100
-        else:
-            # Convert to int for easy adding later
-            pattern['confidence'] = int(pattern['confidence'])
-        app['confidence'][app_type + ' ' + key + pattern['string']] = pattern['confidence']
+        if key != '': key += ' '
+        match_name = app_type + ' ' + key + pattern.string
+        
+        detected_tech.confidence[match_name] = pattern.confidence
 
         # Dectect version number
-        if 'version' in pattern:
-            allmatches = re.findall(pattern['regex'], value)
+        if pattern.version:
+            allmatches = re.findall(pattern.regex, value)
             for i, matches in enumerate(allmatches):
-                version = pattern['version']
-
+                version = pattern.version
                 # Check for a string to avoid enumerating the string
                 if isinstance(matches, str):
                     matches = [(matches)]
@@ -318,34 +264,29 @@ def _set_detected_app(self, app, app_type, pattern, value, key=''):
                     if ternary and len(ternary.groups()) == 2 and ternary.group(1) is not None and ternary.group(2) is not None:
                         version = version.replace(ternary.group(0), ternary.group(1) if match != ''
                                                   else ternary.group(2))
-
                     # Replace back references
                     version = version.replace('\\' + str(index + 1), match)
-                if version != '':
-                    if 'versions' not in app:
-                        app['versions'] = [version]
-                    elif version not in app['versions']:
-                        app['versions'].append(version)
-            self._set_app_version(app)
+                if version != '' and version not in detected_tech.versions:
+                    detected_tech.versions.append(version)
+            self._sort_app_version(detected_tech)
 
-    def _set_app_version(self, app):
+    def _sort_app_version(self, detected_tech: Technology) -> None:
         """
-        Resolve version number (find the longest version number that contains all shorter detected version numbers).
+        Sort version number (find the longest version number that *is supposed to* contains all shorter detected version numbers).
         """
-        if 'versions' not in app:
+        if len(detected_tech.versions) >= 1:
             return
+        detected_tech.versions = sorted(detected_tech.versions, key=self._cmp_to_key(self._sort_app_versions))
 
-        app['versions'] = sorted(app['versions'], key=self._cmp_to_key(self._sort_app_versions))
-
-    def _get_implied_technologies(self, detected_technologies):
+    def _get_implied_technologies(self, detected_technologies:Iterable[str]) -> Iterable[str]:
         """
         Get the set of technologies implied by `detected_technologies`.
         """
-        def __get_implied_technologies(technologies):
+        def __get_implied_technologies(technologies:Iterable[str]) -> Iterable[str] :
             _implied_technologies = set()
             for tech in technologies:
                 try:
-                    for implie in self.technologies[tech]['implies']:
+                    for implie in self.technologies[tech].implies:
                         # If we have no doubts just add technology
                         if 'confidence' not in implie:
                             _implied_technologies.add(implie)
@@ -355,7 +296,7 @@ def __get_implied_technologies(technologies):
                             try:
                                 # Use more strict regexp (cause we have already checked the entry of "confidence")
                                 # Also, better way to compile regexp one time, instead of every time
-                                app_name, confidence = self.confidence_regexp.search(implie).groups()
+                                app_name, confidence = self._confidence_regexp.search(implie).groups() # type: ignore
                                 if int(confidence) >= 50:
                                     _implied_technologies.add(app_name)
                             except (ValueError, AttributeError):
@@ -365,7 +306,7 @@ def __get_implied_technologies(technologies):
             return _implied_technologies
 
         implied_technologies = __get_implied_technologies(detected_technologies)
-        all_implied_technologies = set()
+        all_implied_technologies : Set[str] = set()
 
         # Descend recursively until we've found all implied technologies
         while not all_implied_technologies.issuperset(implied_technologies):
@@ -374,37 +315,42 @@ def __get_implied_technologies(technologies):
 
         return all_implied_technologies
 
-    def get_categories(self, tech_name):
+    def get_categories(self, tech_name:str) -> List[str]:
         """
         Returns a list of the categories for an technology name.
+        :param tech_name: Tech name
         """
-        cat_nums = self.technologies.get(tech_name, {}).get("cats", [])
-        cat_names = [self.categories.get(str(cat_num), "").get("name", "")
-                     for cat_num in cat_nums]
-
+        cat_nums = self.technologies[tech_name].cats if tech_name in self.technologies else []
+        cat_names = [self.categories[str(cat_num)].name
+                     for cat_num in cat_nums if str(cat_num) in self.categories]
         return cat_names
 
-    def get_versions(self, app_name):
+    def get_versions(self, url:str, app_name:str) -> List[str]:
         """
         Retuns a list of the discovered versions for an app name.
+        :param url: URL of the webpage
+        :param app_name: App name
         """
-        return [] if 'versions' not in self.technologies[app_name] else self.technologies[app_name]['versions']
-
-    def get_cpe(self, app_name):
-        """
-        Retuns a list of the discovered cpes for an app name.
-        """
-        return [] if 'cpe' not in self.technologies[app_name] else self.technologies[app_name]['cpe']
+        try:
+            return self.detected_technologies[url][app_name].versions
+        except KeyError:
+            return []
 
-    def get_confidence(self, app_name):
+    def get_confidence(self, url:str, app_name:str) -> Optional[int]:
         """
         Returns the total confidence for an app name.
+        :param url: URL of the webpage
+        :param app_name: App name
         """
-        return [] if 'confidenceTotal' not in self.technologies[app_name] else self.technologies[app_name]['confidenceTotal']
+        try:
+            return self.detected_technologies[url][app_name].confidenceTotal
+        except KeyError:
+            return None
 
-    def analyze(self, webpage):
+    def analyze(self, webpage:IWebPage) -> Set[str]:
         """
-        Return a list of technologylications that can be detected on the web page.
+        Return a set of technology that can be detected on the web page.
+        :param webpage: The Webpage to analyze
         """
         detected_technologies = set()
 
@@ -412,25 +358,27 @@ def analyze(self, webpage):
             if self._has_technology(technology, webpage):
                 detected_technologies.add(tech_name)
 
-        detected_technologies |= self._get_implied_technologies(detected_technologies)
+        detected_technologies.update(self._get_implied_technologies(detected_technologies))
+
         return detected_technologies
 
-    def analyze_with_versions(self, webpage):
+    def analyze_with_versions(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]:
         """
-        Return a list of applications and versions that can be detected on the web page.
+        Return a dict of applications and versions that can be detected on the web page.
+        :param webpage: The Webpage to analyze
         """
         detected_apps = self.analyze(webpage)
         versioned_apps = {}
 
         for app_name in detected_apps:
-            versions = self.get_versions(app_name)
+            versions = self.get_versions(webpage.url, app_name)
             versioned_apps[app_name] = {"versions": versions}
 
         return versioned_apps
-
-    def analyze_with_cpe(self, webpage):
+    
+        def analyze_with_cpe(self, webpage):
         """
-        Return a list of applications and versions that can be detected on the web page.
+        Return a list of applications with their CPEs (Common Platform Enumeration) that can be detected on the web page.
         """
         detected_apps = self.analyze(webpage)
         cpe_detected_apps = {}
@@ -443,7 +391,7 @@ def analyze_with_cpe(self, webpage):
 
     def analyze_with_cpe_and_version(self, webpage):
         """
-        Return a list of applications and versions that can be detected on the web page.
+        Return a list of applications with their CPEs (Common Platform Enumeration) and versions that can be detected on the web page.
         """
         detected_apps = self.analyze(webpage)
         cpe_detected_apps = {}
@@ -464,10 +412,16 @@ def analyze_with_cpe_and_version(self, webpage):
 
         return cpe_detected_apps
 
-
-    def analyze_with_categories(self, webpage):
+    
+    def analyze_with_categories(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]:
         """
-        Return a list of technologies and categories that can be detected on the web page.
+        Return a dict of technologies and categories that can be detected on the web page.
+        :param webpage: The Webpage to analyze
+        >>> wappalyzer.analyze_with_categories(webpage)
+        {'Amazon ECS': {'categories': ['IaaS']},
+        'Amazon Web Services': {'categories': ['PaaS']},
+        'Azure CDN': {'categories': ['CDN']},
+        'Docker': {'categories': ['Containers']}}
         """
         detected_technologies = self.analyze(webpage)
         categorised_technologies = {}
@@ -478,7 +432,19 @@ def analyze_with_categories(self, webpage):
 
         return categorised_technologies
 
-    def analyze_with_versions_and_categories(self, webpage):
+    def analyze_with_versions_and_categories(self, webpage:IWebPage) -> Dict[str, Dict[str, Any]]:
+        """
+        Return a dict of applications and versions and categories that can be detected on the web page.
+        :param webpage: The Webpage to analyze
+        >>> wappalyzer.analyze_with_versions_and_categories(webpage)
+        {'Font Awesome': {'categories': ['Font scripts'], 'versions': ['5.4.2']},
+        'Google Font API': {'categories': ['Font scripts'], 'versions': []},
+        'MySQL': {'categories': ['Databases'], 'versions': []},
+        'Nginx': {'categories': ['Web servers', 'Reverse proxies'], 'versions': []},
+        'PHP': {'categories': ['Programming languages'], 'versions': ['5.6.40']},
+        'WordPress': {'categories': ['CMS', 'Blogs'], 'versions': ['5.4.2']},
+        'Yoast SEO': {'categories': ['SEO'], 'versions': ['14.6.1']}}
+        """
         versioned_apps = self.analyze_with_versions(webpage)
         versioned_and_categorised_apps = versioned_apps
 
@@ -488,10 +454,10 @@ def analyze_with_versions_and_categories(self, webpage):
 
         return versioned_and_categorised_apps
 
-    def _sort_app_versions(self, version_a, version_b):
+    def _sort_app_versions(self, version_a: str, version_b: str) -> int:
         return len(version_a) - len(version_b)
 
-    def _cmp_to_key(self, mycmp):
+    def _cmp_to_key(self, mycmp: Callable[..., Any]):
         """
         Convert a cmp= function into a key= function
         """
@@ -520,3 +486,36 @@ def __ne__(self, other):
                 return mycmp(self.obj, other.obj) != 0
 
         return CmpToKey
+
+def analyze(url:str, 
+            update:bool=False, 
+            useragent:str=None,
+            timeout:int=10,
+            verify:bool=True) -> Dict[str, Dict[str, Any]]:
+    """
+    Quick utility method to analyze a website with minimal configurable options. 
+    :See: `WebPage` and `Wappalyzer`. 
+    :Parameters:
+        - `url`: URL
+        - `update`: Update the technologies file from the internet
+        - `useragent`: Request user agent
+        - `timeout`: Request timeout
+        - `verify`: SSL cert verify
+    
+    :Return: 
+        `dict`. Just as `Wappalyzer.analyze_with_versions_and_categories`. 
+    :Note: More information might be added to the returned values in the future
+    """
+    # Create Wappalyzer
+    wappalyzer=Wappalyzer.latest(update=update)
+    # Create WebPage
+    headers={}
+    if useragent:
+        headers['User-Agent'] = useragent
+    webpage=WebPage.new_from_url(url, 
+        headers=headers, 
+        timeout=timeout, 
+        verify=verify)
+    # Analyze
+    results = wappalyzer.analyze_with_versions_and_categories(webpage)
+    return results