diff --git a/jhack/conf/conf.py b/jhack/conf/conf.py index acb686a..aff0b30 100644 --- a/jhack/conf/conf.py +++ b/jhack/conf/conf.py @@ -58,10 +58,12 @@ def get(self, *path: str) -> bool: # todo: add more toml types? data = data[item] except KeyError: if self._path is self._DEFAULTS: - logger.error(f'{item} not found in default config; invalid path') + logger.error(f"{item} not found in default config; invalid path") raise - logger.info(f'{item} not found in user-config {self._path}; defaulting...') + logger.info( + f"{item} not found in user-config {self._path}; defaulting..." + ) return self.get_default(*path) return data diff --git a/jhack/helpers.py b/jhack/helpers.py index dc861cd..b2bc918 100644 --- a/jhack/helpers.py +++ b/jhack/helpers.py @@ -39,6 +39,14 @@ def check_command_available(cmd: str): return proc.returncode == 0 +def get_current_controller() -> str: + cmd = f"juju whoami --format=json" + proc = JPopen(cmd.split()) + raw = proc.stdout.read().decode("utf-8") + whoami_info = jsn.loads(raw) + return whoami_info["controller"] + + def get_substrate(model: str = None) -> Literal["k8s", "machine"]: """Attempts to guess whether we're talking k8s or machine.""" cmd = f'juju show-model{f" {model}" if model else ""} --format=json' diff --git a/jhack/mongo/eson.py b/jhack/mongo/eson.py new file mode 100644 index 0000000..b4ad0ca --- /dev/null +++ b/jhack/mongo/eson.py @@ -0,0 +1,54 @@ +def emit_thing(v): + if v.__class__.__name__ == "dict": + return emit_dict(v) + elif v.__class__.__name__ == "list": + return emit_list(v) + elif v.__class__.__name__ in { + "Int64", + "int", + "long", + "float", + "decimal", + "Decimal128", + "Decimal", + }: + return str(v) + elif v.__class__.__name__ == "datetime": + return v + else: + return str(v) + + +def emit_list(ll: list) -> list: + return list(map(emit_thing, ll)) + + +def emit_dict(dd: dict) -> dict: + out = {} + for k, v in dd.items(): + out[k] = emit_thing(v) + return out + + +def parse_eson(doc: str) -> dict: + return emit_dict(doc) + + +import json +import re + +from bson import json_util + + +def read_mongoextjson_file(filename): + with open(filename, "r") as f: + bsondata = f.read() + # Convert Mongo object(s) to regular strict JSON + jsondata = re.sub( + r"ObjectId\s*\(\s*\"(\S+)\"\s*\)", r'{"$oid": "\1"}', bsondata + ) + # Description of Mongo ObjectId: + # https://docs.mongodb.com/manual/reference/mongodb-extended-json/#mongodb-bsontype-ObjectId + # now we can parse this as JSON, and use MongoDB's object_hook + data = json.loads(jsondata, object_hook=json_util.object_hook) + return data diff --git a/jhack/mongo/get_credentials_from_k8s_controller.sh b/jhack/mongo/get_credentials_from_k8s_controller.sh new file mode 100644 index 0000000..641addf --- /dev/null +++ b/jhack/mongo/get_credentials_from_k8s_controller.sh @@ -0,0 +1,8 @@ +#!/bin/bash +kubectl_bin=/snap/bin/microk8s.kubectl +k8s_ns=$(juju whoami | grep Controller | awk '{print "controller-"$2}') +k8s_controller_pod=$(${kubectl_bin} -n "${k8s_ns}" get pods | grep -E "^controller-([0-9]+)" | awk '{print $1}') +mongo_user=$(${kubectl_bin} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c api-server -it -- bash -c "grep tag /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'") +mongo_pass=$(${kubectl_bin} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c api-server -it -- bash -c "grep statepassword /var/lib/juju/agents/controller-*/agent.conf | cut -d' ' -f2 | tr -d '\n'") + +echo "$kubectl_bin" "$mongo_user" "$mongo_pass" "$k8s_ns" "$k8s_controller_pod" diff --git a/jhack/mongo/get_credentials_from_machine_controller.sh b/jhack/mongo/get_credentials_from_machine_controller.sh new file mode 100644 index 0000000..229d768 --- /dev/null +++ b/jhack/mongo/get_credentials_from_machine_controller.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +machine=${1} +model=${2} + +read -d '' -r cmds <<'EOF' +conf=/var/lib/juju/agents/machine-*/agent.conf +user=`sudo grep tag $conf | cut -d' ' -f2` +password=`sudo grep statepassword $conf | cut -d' ' -f2` +if [ -f /snap/bin/juju-db.mongo ]; then + client=/snap/bin/juju-db.mongo +elif [ -f /usr/lib/juju/mongo*/bin/mongo ]; then + client=/usr/lib/juju/mongo*/bin/mongo +else + client=/usr/bin/mongo +fi +echo "$client" "$user" "$password" +EOF +juju ssh -m "${model}" "${machine}" "${cmds}" diff --git a/jhack/mongo/mongo.py b/jhack/mongo/mongo.py new file mode 100644 index 0000000..47717e6 --- /dev/null +++ b/jhack/mongo/mongo.py @@ -0,0 +1,147 @@ +#!/bin/bash +import json +import os +import re +import shlex +from pathlib import Path +from subprocess import PIPE, Popen +from typing import List, Literal, Tuple + +from jhack.helpers import JPopen, get_current_model, get_substrate + + +def escape_double_quotes(query): + return query.replace('"', r"\"") + + +def numberlong(s: str): + return re.sub(r"NumberLong\((\d+)\)", r"\1", s) + + +FILTERS = [numberlong] + + +def to_json(query_result: str): + jsn_str = query_result + for f in FILTERS: + jsn_str = f(jsn_str) + return json.loads(jsn_str) + + +class TooManyResults(RuntimeError): + """Raised when a query returns more results than we handle.""" + + +class EmptyQueryResult(RuntimeError): + """Query returned no results.""" + + +class ConnectorBase: + args_getter_script: Path + query_script: Path + + def __init__(self, controller: str = None, unit_id: int = 0): + self.controller = controller + self.model = f"{controller}:controller" if controller else "controller" + self.unit_id = unit_id + self.args = self.get_args() + + def _escape_query(self, query: str) -> str: + return rf'"{escape_double_quotes(query)}"' + + def _get_output(self, cmd: str) -> str: + return JPopen(shlex.split(cmd)).stdout.read().decode("utf-8") + + def get_args(self) -> Tuple[str, ...]: + out = ( + Popen( + shlex.split( + f"bash {self.args_getter_script.absolute()} {self.unit_id} {self.model}", + ), + stdout=PIPE, + ) + .stdout.read() + .decode("utf-8") + ) + return tuple(out.split()) # noqa + # return tuple(f"'{x}'" for x in out.split()) # noqa + + def get(self, query: str, n: int = None, query_filter: str = None) -> List[dict]: + qfilter = query_filter or "{}" + if n is None: + q = query + fr".find({qfilter}).toArray()" + else: + q = query + fr".find({qfilter}).limit({n}).toArray()" + escaped = self._escape_query(q) + out = self._run_query(escaped) + + if not out: + raise EmptyQueryResult(escaped) + return out + + def _run_query(self, query: str): + command = ["bash", str(self.query_script.absolute()), *self.args, query] + proc = Popen(command, stdout=PIPE, stderr=PIPE) + raw_output = proc.stdout.read().decode("utf-8") + if not raw_output: + err = proc.stderr.read().decode("utf-8") + print(err) + raise RuntimeError(f"unexpected result from command {command}; {err!r}") + + # stripped = "[" + "\n".join(filter(None, raw_output.split('\n')[1:])) + "]" + stripped = "\n".join(filter(None, raw_output.split('\n')[1:])) + try: + return to_json(stripped) + except Exception as e: + err = proc.stderr.read().decode("utf-8") + print(err) + raise RuntimeError( + f"failed deserializing query result {stripped} with {type(e)} {err}" + ) from e + + +class K8sConnector(ConnectorBase): + """Mongo database connector for kubernetes controllers.""" + + args_getter_script = ( + Path(__file__).parent / "get_credentials_from_k8s_controller.sh" + ) + query_script = Path(__file__).parent / "query_k8s_controller.sh" + + +class MachineConnector(ConnectorBase): + """Mongo database connector for kubernetes controllers.""" + + args_getter_script = ( + Path(__file__).parent / "get_credentials_from_machine_controller.sh" + ) + query_script = Path(__file__).parent / "query_machine_controller.sh" + + def get_args(self): + return super().get_args() + (self.model, str(self.unit_id)) + + +class Mongo: + def __init__( + self, + entity_id: int = 0, + substrate: Literal["k8s", "machine"] = None, + model: str = None, + ): + self.substrate = substrate or get_substrate() + self.entity_id = entity_id + self.model = model or get_current_model() + + if substrate == "k8s": + self.connector = K8sConnector() + + elif substrate == "machine": + self.connector = MachineConnector() + + else: + raise TypeError(substrate) + + def _get(self, query: str, n: int = None, query_filter: str = None): + return self.connector.get(query, n=n, query_filter=query_filter) + + diff --git a/jhack/mongo/query_k8s_controller.sh b/jhack/mongo/query_k8s_controller.sh new file mode 100644 index 0000000..056b211 --- /dev/null +++ b/jhack/mongo/query_k8s_controller.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -x +kctl=${1} +user=${2} +password=${3} +k8s_ns=${4} +k8s_controller_pod=${5} +query=${6} +#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --ssl --sslAllowInvalidCertificates --username '${user}' --password '${password}' --help" +#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username '${user}' --password '${password}' --eval '${query}'" +microk8s.kubectl exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -t -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password} --eval ${query}" +#${kctl} exec -n "${k8s_ns}" "${k8s_controller_pod}" -c mongodb -it -- bash -c "/bin/mongo 127.0.0.1:37017/juju --authenticationDatabase admin --quiet --tls --tlsAllowInvalidCertificates --username ${user} --password ${password}" diff --git a/jhack/mongo/query_machine_controller.sh b/jhack/mongo/query_machine_controller.sh new file mode 100644 index 0000000..42a94ed --- /dev/null +++ b/jhack/mongo/query_machine_controller.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -x +client=${1} +user=${2} +password=${3} +model=${4} +machine=${5} +query=${6} +juju ssh -m "$model" "$machine" -- "$client" '127.0.0.1:37017/juju' --authenticationDatabase admin --tls --tlsAllowInvalidCertificates --quiet --username "$user" --password "$password" --eval "$query" diff --git a/jhack/tests/config/test_config.py b/jhack/tests/config/test_config.py index 20cb6a2..6ac1653 100644 --- a/jhack/tests/config/test_config.py +++ b/jhack/tests/config/test_config.py @@ -64,4 +64,3 @@ def test_defaults(): assert cfg.get("test", "bar") == "baz" Config._DEFAULTS = old_def - diff --git a/jhack/tests/mongo/test_k8s_connector_manual.py b/jhack/tests/mongo/test_k8s_connector_manual.py new file mode 100644 index 0000000..00da8ca --- /dev/null +++ b/jhack/tests/mongo/test_k8s_connector_manual.py @@ -0,0 +1,22 @@ +from jhack.mongo.mongo import K8sConnector + + +def test_k8s_connector_base(): + connector = K8sConnector() + query = r"db.relations" + val = connector.get(query, n=1) + assert len(val) == 1 + + +def test_k8s_connector_get_all(): + connector = K8sConnector() + query = r"db.relations" + val = connector.get(query) + assert len(val) == 26 + + +def test_k8s_connector_relation(): + connector = K8sConnector() + query = r'db.relations' + val = connector.get(query, query_filter='{"key": "grafana:catalogue catalogue:catalogue"}') + assert len(val) == 1 \ No newline at end of file diff --git a/jhack/tests/mongo/test_machine_connector_manual.py b/jhack/tests/mongo/test_machine_connector_manual.py new file mode 100644 index 0000000..eec9e15 --- /dev/null +++ b/jhack/tests/mongo/test_machine_connector_manual.py @@ -0,0 +1,15 @@ +from jhack.mongo.mongo import MachineConnector + + +def test_machine_connector_base(): + connector = MachineConnector("lxdcloud") + query = r"db.relations" + val = connector.get(query, n=1) + assert len(val) == 1 + + +def test_machine_connector(): + connector = MachineConnector("lxdcloud") + query = 'db.relations' + val = connector.get(query, query_filter='{"key": "kafka:cluster"}') + assert len(val) == 1