diff --git a/Run_TCAV.ipynb b/Run_TCAV.ipynb index b1204de5b3..1df0b2bcbc 100644 --- a/Run_TCAV.ipynb +++ b/Run_TCAV.ipynb @@ -1,29 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "kernelspec": { - "display_name": "Python 3.7.7 64-bit ('tf1.15': conda)", - "language": "python", - "name": "python_defaultSpec_1594677056398" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7-final" - }, - "colab": { - "name": "Run TCAV.ipynb", - "provenance": [] - } - }, "cells": [ { "cell_type": "markdown", @@ -56,16 +31,92 @@ }, { "cell_type": "code", + "execution_count": 1, "metadata": { - "tags": [], - "id": "sIHww5CuYYdk" + "id": "sIHww5CuYYdk", + "tags": [] }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mDEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\n", + "Requirement already satisfied: tensorflow in /usr/local/lib/python3.9/site-packages (2.9.1)\n", + "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.6.3)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/site-packages (from tensorflow) (60.2.0)\n", + "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.1.2)\n", + "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.9/site-packages (from tensorflow) (3.19.4)\n", + "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (3.7.0)\n", + "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.9/site-packages (from tensorflow) (4.0.1)\n", + "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (14.0.6)\n", + "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.2.0)\n", + "Requirement already satisfied: keras<2.10.0,>=2.9.0rc0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (2.9.0)\n", + "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.1.0)\n", + "Requirement already satisfied: tensorflow-estimator<2.10.0,>=2.9.0rc0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (2.9.0)\n", + "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.14.1)\n", + "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.9/site-packages (from tensorflow) (3.3.0)\n", + "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.9/site-packages (from tensorflow) (0.2.0)\n", + "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.9/site-packages (from tensorflow) (0.4.0)\n", + "Requirement already satisfied: tensorboard<2.10,>=2.9 in /usr/local/lib/python3.9/site-packages (from tensorflow) (2.9.1)\n", + "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.22.0)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.9/site-packages (from tensorflow) (21.3)\n", + "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.47.0)\n", + "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.16.0)\n", + "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.9/site-packages (from tensorflow) (0.26.0)\n", + "Requirement already satisfied: flatbuffers<2,>=1.12 in /usr/local/lib/python3.9/site-packages (from tensorflow) (1.12)\n", + "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.9/site-packages (from astunparse>=1.6.0->tensorflow) (0.37.1)\n", + "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (2.26.0)\n", + "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (0.4.6)\n", + "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (2.11.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (3.4.1)\n", + "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (1.8.1)\n", + "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (2.2.2)\n", + "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.9/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (0.6.1)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.9/site-packages (from packaging->tensorflow) (3.0.6)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (0.2.8)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (4.7.2)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (5.2.0)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.9/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow) (1.3.1)\n", + "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.9/site-packages (from markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow) (4.12.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (3.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (2021.10.8)\n", + "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (2.0.9)\n", + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (1.26.7)\n", + "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.9/site-packages (from werkzeug>=1.0.1->tensorboard<2.10,>=2.9->tensorflow) (2.1.1)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow) (3.8.1)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (0.4.8)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.9/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow) (3.2.0)\n", + "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.2.2 is available.\n", + "You should consider upgrading via the '/usr/local/opt/python@3.9/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n", + "\u001b[33mDEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\n", + "Requirement already satisfied: tcav in /usr/local/lib/python3.9/site-packages (0.2.2)\n", + "Requirement already satisfied: Pillow>=6.0.0 in /usr/local/lib/python3.9/site-packages (from tcav) (9.2.0)\n", + "Requirement already satisfied: scipy>=1.2.1 in /usr/local/lib/python3.9/site-packages (from tcav) (1.9.1)\n", + "Requirement already satisfied: protobuf>=3.10.0 in /usr/local/lib/python3.9/site-packages (from tcav) (3.19.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.3 in /usr/local/lib/python3.9/site-packages (from tcav) (1.1.2)\n", + "Requirement already satisfied: matplotlib>=2.2.4 in /usr/local/lib/python3.9/site-packages (from tcav) (3.5.3)\n", + "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (1.22.0)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (2.8.2)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (0.11.0)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (4.37.1)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (1.4.4)\n", + "Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (3.0.6)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/site-packages (from matplotlib>=2.2.4->tcav) (21.3)\n", + "Requirement already satisfied: joblib>=1.0.0 in /usr/local/lib/python3.9/site-packages (from scikit-learn>=0.20.3->tcav) (1.1.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.9/site-packages (from scikit-learn>=0.20.3->tcav) (3.1.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib>=2.2.4->tcav) (1.16.0)\n", + "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.2.2 is available.\n", + "You should consider upgrading via the '/usr/local/opt/python@3.9/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install tensorflow\n", "%pip install tcav" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -106,21 +157,23 @@ }, { "cell_type": "code", + "execution_count": 2, "metadata": { "id": "sS1ZjSZjYYdl" }, + "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": 3, "metadata": { "id": "U4yP9kDlYYdl" }, + "outputs": [], "source": [ "import tcav.activation_generator as act_gen\n", "import tcav.cav as cav\n", @@ -130,9 +183,7 @@ "import tcav.utils_plot as utils_plot # utils_plot requires matplotlib\n", "import os \n", "import tensorflow as tf" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -182,16 +233,26 @@ }, { "cell_type": "code", + "execution_count": 4, "metadata": { - "tags": [], - "id": "EHabMnMNYYdm" + "id": "EHabMnMNYYdm", + "tags": [] }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "REMEMBER TO UPDATE YOUR_PATH (where images, models are)!\n" + ] + } + ], "source": [ "print ('REMEMBER TO UPDATE YOUR_PATH (where images, models are)!')\n", "\n", "# This is the name of your model wrapper (InceptionV3 and GoogleNet are provided in model.py)\n", "model_to_run = 'GoogleNet' \n", - "user = 'beenkim'\n", + "user = 'mariaf'\n", "# the name of the parent directory that results are stored (only if you want to cache)\n", "project_name = 'tcav_class_test'\n", "working_dir = \"/tmp/\" + user + '/' + project_name\n", @@ -203,8 +264,8 @@ "# where the images live.\n", "\n", "# TODO: replace 'YOUR_PATH' with path to downloaded models and images. \n", - "source_dir = ''\n", - "bottlenecks = [ 'mixed4c'] # @param \n", + "source_dir = '/Users/maria/MastersThesis/tcav/ImageNet_Data'\n", + "bottlenecks = ['mixed3a'] # @param \n", " \n", "utils.make_dir_if_not_exists(activation_dir)\n", "utils.make_dir_if_not_exists(working_dir)\n", @@ -215,9 +276,7 @@ "\n", "target = 'zebra' \n", "concepts = [\"dotted\",\"striped\",\"zigzagged\"] \n" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -288,10 +347,21 @@ }, { "cell_type": "code", + "execution_count": 5, "metadata": { - "tags": [], - "id": "hH-YQiEIYYdn" + "id": "hH-YQiEIYYdn", + "tags": [] }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-09-07 13:20:47.066892: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F AVX512_VNNI FMA\n", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" + ] + } + ], "source": [ "# Create TensorFlow session.\n", "sess = utils.create_session()\n", @@ -311,9 +381,7 @@ "mymodel = model.GoogleNetWrapper_public(sess,\n", " GRAPH_PATH,\n", " LABEL_PATH)" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -337,14 +405,14 @@ }, { "cell_type": "code", + "execution_count": 6, "metadata": { "id": "ZmSyFxQbYYdo" }, + "outputs": [], "source": [ "act_generator = act_gen.ImageActivationGenerator(mymodel, source_dir, activation_dir, max_examples=100)" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -363,15 +431,93 @@ }, { "cell_type": "code", + "execution_count": 17, "metadata": { + "id": "F2FVOGSvYYdp", "scrolled": false, - "tags": [], - "id": "F2FVOGSvYYdp" + "tags": [] }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:mixed3a ['dotted', 'random500_0'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['dotted', 'random500_1'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['dotted', 'random500_2'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['dotted', 'random500_3'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['striped', 'random500_0'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['striped', 'random500_1'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['striped', 'random500_2'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['striped', 'random500_3'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['zigzagged', 'random500_0'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['zigzagged', 'random500_1'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['zigzagged', 'random500_2'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['zigzagged', 'random500_3'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_0', 'random500_1'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_0', 'random500_2'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_0', 'random500_3'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_1', 'random500_0'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_1', 'random500_2'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_1', 'random500_3'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_2', 'random500_0'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_2', 'random500_1'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_2', 'random500_3'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_3', 'random500_0'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_3', 'random500_1'] zebra 0.1\n", + "INFO:tensorflow:mixed3a ['random500_3', 'random500_2'] zebra 0.1\n", + "INFO:tensorflow:TCAV will 24 params\n", + "This may take a while... Go get coffee!\n", + "INFO:tensorflow:running 24 params\n", + "INFO:tensorflow:Running param 0 of 24\n", + "INFO:tensorflow:running zebra ['dotted', 'random500_0']\n", + "INFO:tensorflow:/tmp/mariafogh/tcav_class_test/activations/acts_dotted_mixed3a does not exist, Making one...\n", + "INFO:tensorflow:/tmp/mariafogh/tcav_class_test/activations/acts_random500_0_mixed3a does not exist, Making one...\n", + "INFO:tensorflow:cannot identify image file \n", + "INFO:tensorflow:/tmp/mariafogh/tcav_class_test/activations/acts_zebra_mixed3a does not exist, Making one...\n", + "INFO:tensorflow:Training CAV ['dotted', 'random500_0'] - mixed3a alpha 0.1\n", + "INFO:tensorflow:training with alpha=0.1\n", + "INFO:tensorflow:understand output min_data_points = 40\n", + "INFO:tensorflow:understand output concept = dotted\n", + "INFO:tensorflow:understand output bottleneck = mixed3a\n", + "INFO:tensorflow:understand output acts.keys() = dict_keys(['dotted', 'random500_0'])\n", + "INFO:tensorflow:understand output acts[concept][bottleneck].shape[0] = 50\n", + "INFO:tensorflow:understand output min_data_points = 40\n", + "INFO:tensorflow:understand output concept = random500_0\n", + "INFO:tensorflow:understand output bottleneck = mixed3a\n", + "INFO:tensorflow:understand output acts.keys() = dict_keys(['dotted', 'random500_0'])\n", + "INFO:tensorflow:understand output acts[concept][bottleneck].shape[0] = 40\n", + "INFO:tensorflow:acc per class {'dotted': 0.7692307692307693, 'random500_0': 0.9285714285714286, 'overall': 0.8518518518518519}\n", + "INFO:tensorflow:CAV accuracies: {'dotted': 0.7692307692307693, 'random500_0': 0.9285714285714286, 'overall': 0.8518518518518519}\n", + "INFO:tensorflow:cannot identify image file \n", + "INFO:tensorflow:cannot identify image file \n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/bx/1rzxld1d3pdfks1bqj8dn3jw0000gn/T/ipykernel_11873/144921439.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 12\u001b[0m num_random_exp=num_random_exp)#10)\n\u001b[1;32m 13\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'This may take a while... Go get coffee!'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmytcav\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_parallel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'done!'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/MastersThesis/tcav/tcav/tcav.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, num_workers, run_parallel, overwrite, return_proto)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0;31m# tf.compat.v1.logging.info(\"Hello: We are at %s\" % (vars(param))) #print parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m results.append(\n\u001b[0;32m--> 257\u001b[0;31m self._run_single_set(\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_parallel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrun_parallel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m )\n", + "\u001b[0;32m~/MastersThesis/tcav/tcav/tcav.py\u001b[0m in \u001b[0;36m_run_single_set\u001b[0;34m(self, param, overwrite, run_parallel)\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0mcav_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0macts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtarget_class\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcav_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbottleneck\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m \u001b[0mactivation_generator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_examples_for_concept\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_class\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 336\u001b[0m )\n\u001b[1;32m 337\u001b[0m result = {\n", + "\u001b[0;32m~/MastersThesis/tcav/tcav/activation_generator.py\u001b[0m in \u001b[0;36mget_examples_for_concept\u001b[0;34m(self, concept)\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcept_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcept_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m ]\n\u001b[0;32m--> 118\u001b[0;31m imgs = self.load_images_from_files(\n\u001b[0m\u001b[1;32m 119\u001b[0m img_paths, self.max_examples, shape=self.model.get_image_shape()[:2])\n\u001b[1;32m 120\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mimgs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/MastersThesis/tcav/tcav/activation_generator.py\u001b[0m in \u001b[0;36mload_images_from_files\u001b[0;34m(self, filenames, max_imgs, do_shuffle, run_parallel, shape, num_workers)\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_parallel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 188\u001b[0m \u001b[0mpool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmultiprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_workers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 189\u001b[0;31m imgs = pool.map(\n\u001b[0m\u001b[1;32m 190\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_image_from_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m filenames[:max_imgs])\n", + "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, func, iterable, chunksize)\u001b[0m\n\u001b[1;32m 362\u001b[0m \u001b[0;32min\u001b[0m \u001b[0ma\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mthat\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mreturned\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 363\u001b[0m '''\n\u001b[0;32m--> 364\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_map_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapstar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 366\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstarmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 763\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 765\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 766\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 767\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 572\u001b[0m \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 573\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 574\u001b[0;31m \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 575\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 576\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/python@3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 312\u001b[0;31m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 313\u001b[0m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 314\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "import absl\n", "absl.logging.set_verbosity(0)\n", - "num_random_exp=3\n", + "num_random_exp=4\n", "## only running num_random_exp = 10 to save some time. The paper number are reported for 500 random runs. \n", "mytcav = tcav.TCAV(sess,\n", " target,\n", @@ -384,52 +530,79 @@ "print ('This may take a while... Go get coffee!')\n", "results = mytcav.run(run_parallel=False)\n", "print ('done!')" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": 9, "metadata": { - "scrolled": false, - "tags": [], "id": "hjKVKa80YYdp", - "outputId": "ff3cbebe-4edd-4342-cede-24dc10ce9d99" + "outputId": "ff3cbebe-4edd-4342-cede-24dc10ce9d99", + "scrolled": false, + "tags": [] }, - "source": [ - "utils_plot.plot_results(results, num_random_exp=num_random_exp)" - ], - "execution_count": null, "outputs": [ { + "name": "stdout", "output_type": "stream", "text": [ "Class = zebra\n", - " Concept = dotted\n", - " Bottleneck = mixed4c. TCAV Score = 0.56 (+- 0.12), random was 0.54 (+- 0.16). p-val = 0.917 (not significant)\n", - " Concept = striped\n", - " Bottleneck = mixed4c. TCAV Score = 0.89 (+- 0.09), random was 0.54 (+- 0.16). p-val = 0.018 (significant)\n", - " Concept = zigzagged\n", - " Bottleneck = mixed4c. TCAV Score = 0.84 (+- 0.10), random was 0.54 (+- 0.16). p-val = 0.036 (significant)\n", - "{'mixed4c': {'bn_vals': [0.01, 0.888888888888889, 0.8376068376068376], 'bn_stds': [0, 0.08716272672808177, 0.09669836323918596], 'significant': [False, True, True]}}\n" - ], - "name": "stdout" + " Concept = dotted\n" + ] }, { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAacAAAEYCAYAAAD4czk4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df7wWZZ3/8dcbAREklR+WCAopihqr0Unli26U4qIWuJsWhBVmuLXSZmXKtpuZaz4if2yb6SqVkpWgtJqk+AuV/FEqmKSJQichPRgGqAW6iOLn+8dcB4fjue9zHzz3ueec834+HvfjzFxzzTWfmXvO/bln5rpnFBGYmZkVSbdaB2BmZtaUk5OZmRWOk5OZmRWOk5OZmRWOk5OZmRWOk5OZmRWOk5MVmqSdJP1S0l8lzat1PKVIWiXp6FrH0dFJOlfST0tMGyupocrLr+n7KGm2pPNrtfwicXJqB5I25l5vSPq/3PgUSe+Q9F1Jz6SyP6bxAU3aWSTpRUk7pvHDJb0saedmlvmopOkl4vmapJVpWQ2SrqvOmreJE4F3Av0j4qRaB9PVlEsWHYmkqZLub1LmRFBgTk7tICJ2bnwBzwAfyY3PA+4CDgLGA+8ARgPrgUMb25A0FDgSCGBCavdBoIHsA5xc3fcABwJzmsYi6dPAJ4Gj0/Lr0vLbjKTubdjc3sCKiHi9xnGYWXuKCL/a8QWsIksMjeOfBZ4Hdm5hvnOAB4BLgJtz5V8D7m5S9zvAjSXa+T7w3TLL6QdcDTwHvAj8IjdtGlAPvADMBwblpgVwOvAHYGUq+zCwFHgJ+DXwd7n6ZwOrgQ3AcuCoZmL5JrAZeA3YCJxK9oXqP4A/AX8BrgF2SfWHpjhOJfsScG+JdSwX1wzgjymuZcA/Npl3GvBkbvqo3Pt6JvAY8FfgOqBXme1cqp0DgEUptieACbl5ZgOXAbek+R4C9slNPwi4M70/zwNfS+Xdcuu1Hrge6Ndkm52W3vM/A2emaeObbP/flViXktsMmArcD1xEtj+tBI7NTR8G/CrNeyfZ/vnTEssZS/Zl7GvAurTNp+Sm75L2h7Vp//iPtO4HAJuALWk9Xkrr+1pav43AL5v+f1a43T5Ntq+tA/49F0vJedP0I8j2vZeAZ4Gpuff4/DTcF7gH+B6gWn92tfer5gF0tRdvTU5zgR9XMF898C/A+9I/1TtT+RDgdWBIGu+W/oFPKNHOyWQfXl8lO2raocn0W8g+WHcDegAfSOUfSv+Ao4AdgUvJffinf9Q7yZLbTsB7yZLHYcAO6Z94VZp3//QPOSjNO5Tch2yTeM4l92EFfCZti3cDOwM3AD/JtRNkH1B9gJ2aaa9kXGn6ScCgtB0/DrwM7JGbthp4PyBgX2Dv3Pv6cJq3H1ni+VyJdWq2nbS968k+fHumbb4B2D/NN5s3j6i7Az8D5qZpfckSy1eAXmn8sDTti8CDwOC0/a8E5jTZZnPSNhtJ9uF+dHPbv8z6lNpmU8n212lpe3+eLAkqTf8N2ReuHYG/T+tbLjm9nqv/gbSsxu1zDXBTWvehwArg1Fwc9zdpbzYpETT3/1nhdvsB2f5+MPAqcEAF8+6d1nNyes/7A4fkY0plDzeNryu9ah5AV3vx1uR0J/DtFuY5Iv2DD0jjTwFfyk1fyJvfkselD5ceZdqbkuZ5mezD7uxUvgfwBrBbM/P8CPhObnznFNPQNB7Ah3LT/wf4zyZtLE8fKPuSJYijy8WZ5jmXbZPTXcC/5Mb3T3F0z31gvLtMeyXjKlF/KTAxDd8OfLHM+3pybvw7wBUl6jbbDtlp2zVAt1zZHODcNDwb+GFu2nHAU2l4MvBoieU9Se7INL3PTbfZiCax/6i57V/hPp7fZlOB+ty03ml57wL2Iks2fXLTry21PN5MTvn61wNfJ0t8m4EDc9P+GViUi6O1yamS7TY4N/1hYFIF8/4bpc9szAauAn4PfLU1272zvXzNqfbWk+245XwauCMi1qXxa1NZox+TXUci/Z0bEa+VaiwifhYRRwO7Ap8D/lPSP5Adhb0QES82M9sgslMljW1sTLHvmavzbG54b+Arkl5qfKX2B0VEPXAG2QffXyTNlTSo7BYoEUca7k7WaaK5OJoqGReApE9JWpqb9h6gsWPKELLTNKWsyQ2/QpbAm1OqnUHAsxHxRq7sT2y7jUsto1xsewM35tbpSbJTXKW22Z9SLBVpYZttE3NEvJIGd07LeDEiXm6y7HKaqz8oLa8Hb9038tuutSrZbqXej3LztrQfHU92NHbF24i9w3Nyqr2FwD9I6tPcREk7AR8DPiBpjaQ1wJeAgyUdnKrdAAyW9EHgn8iSVYsi4rWImEd2neQ9ZB9Q/STt2kz158j+4Rrj6kN26mF1vsnc8LPAtyJi19yrd0TMScu+NiKOSG0GMLOSmJvGwZvfvp8vEUdTJeOStDfZaZrpZL0DdyX7BqvcvPtUGGc5pdp5DhgiKf9/uRfbbuNybb67zLRjm6xzr4jItzukyTKfS8PltiUVbLNy/gzs1mTf36uFeZqr/xzZKefXeOu+0biOza1H2XWjsu22PfO2tB/9ALgNWFDqc6ErcHKqvZ+Q7az/K2mEpG6S+qfu3scBJ5B94zoQOCS9DgDuAz4FkL5J/pysI8OfImJJqYWlLrXHS+qblnUs2YX0hyLiz8CtwOWSdpPUQ9Lfp1nnAKdIOiR1Zb8gzbOqxKJ+AHxO0mHK9Mktd39JH0rtbAL+j+x0YiXmAF+SNCx1ob8AuC4q781XMi6yay5BdloUSaeQJe1GPwTOlPS+NO++6cO5tUq18xDZt++z0rYfC3yE7LpkS24G9pB0hqQd03Y+LE27AvhWY6ySBkqa2GT+r0vqLekg4BSy646QJf2hTRJmXkvbrKSI+BOwBPimpJ6Sjkjr25LG+keSdW6ZFxFbyE7xfSut+97Al4HGbvDPk32B65lr53lKJ3SobLttz7w/A46W9DFJ3dP/+yFN5p9Odrr5l+kLapfj5FRjEfEq2bWXp8iuP/2N7Nz1ALIPq08DV0fEMxGxpvFF1qtpSq679I/JvjVe08Ii/0Z2wf0Zsp5C3wE+HxGNvwH5JNk30KfIrgudkeJcSHZu/3/JvvHuA0wqs15LyC6Cf5+sl1Y92Xl/yC4Qf5vs2+4aYHey8/CVuIosod9L1vNrE/CFCuctG1dELAMuJrtI/zxZ54AHcvPOA75Fdlp1A/ALss4PrVKqnYjYTPbhfCzZtrkc+FREPFVBmxvIrjd+hGyb/gH4YJr832S9K++QtIHsQv1hTZr4Fdm2uAu4KCLuSOWNP3xeL+m3zSy37DarwCdSLC8A36Dl/XcN2fv2HNmH/Ody2+cLZNdRnybrIXgt2f4CcDdZ78c1khpPj/8IODCdevtFM8uqZLuVUnLeiHiG7HrhV9J6LyXrULFVRARZj8IG4CZJvSpcbqfR2GPGzLogZb+fW0nWMaXVvyUzqxYfOZmZWeE4OZmZWeH4tJ6ZmRWOj5zMzKxwOtyNMQcMGBBDhw6tdRhmZlbGI488si4iBm7v/B0uOQ0dOpQlS0r+jMfMzApAUkt3+yjLp/XMzKxwqpacJF0l6S+Sfl9iuiR9T1K9pMckjapWLGZm1rFU88hpNtnzYEo5FhieXqeR3S3azMysetecIuLe9OvzUiYC16TbdDwoaVdJe6T7u5mZtbvXXnuNhoYGNm3aVOtQOoxevXoxePBgevTo0abt1rJDxJ5se5v+hlT2luQk6TSyoyv22qulmxabmW2fhoYG+vbty9ChQ5EqubF61xYRrF+/noaGBoYNG9ambXeIDhERMSsi6iKibuDA7e6ZaGZW1qZNm+jfv78TU4Uk0b9//6ocadYyOa1m22fIDKay59aYmVWNE1PrVGt71TI5zQc+lXrtHQ781debzMwMqnjNSdIcYCwwQFID2bNaegBExBXAArJnmtSTPWDtlGrFYma2PYbOuKVN21v17ePbpJ358+ezbNkyZsyY8bbaWbRoERdddBE333zz1rLFixczevRo5s6dy4knnvh2Q91u1eytN7mF6QGcXq3lm5l1VhMmTGDChAlt3u6WLVs4++yzOeaYY9q87dbqEB0izDqzsWPHMnbs2FqHYQWxatUqRowYwdSpU9lvv/2YMmUKCxcuZMyYMQwfPpyHH36Y2bNnM336dAAmTpzINddkDxC+8sormTJlCgB33HEHo0ePZtSoUZx00kls3LgRgNtuu40RI0YwatQobrjhhm2Wfemll/LRj36U3XfffZvymTNnMnLkSA4++OC3fbRWqQ53bz0zs86uvr6eefPmcdVVV/H+97+fa6+9lvvvv5/58+dzwQUXcMIJJ2ytO2vWLMaMGcOwYcO4+OKLefDBB1m3bh3nn38+CxcupE+fPsycOZNLLrmEs846i2nTpnH33Xez77778vGPf3xrO6tXr+bGG2/knnvuYfHixVvLb731Vm666SYeeughevfuzQsvvNAu28DJycysYIYNG8bIkSMBOOiggzjqqKOQxMiRI1m1atU2dd/5zndy3nnn8cEPfpAbb7yRfv36cfPNN7Ns2TLGjBkDwObNmxk9ejRPPfUUw4YNY/jw4QCcfPLJzJo1C4AzzjiDmTNn0q3btifUFi5cyCmnnELv3r0B6NevXzVXfSsnJzOzgtlxxx23Dnfr1m3reLdu3Xj99dffUv/xxx+nf//+PPfcc0D249hx48YxZ86cbeotXbq05DKXLFnCpEmTAFi3bh0LFiyge/fapQhfczIz68Aefvhhbr31Vh599FEuuugiVq5cyeGHH84DDzxAfX09AC+//DIrVqxgxIgRrFq1ij/+8Y8A2ySvlStXsmrVKlatWsWJJ57I5ZdfzgknnMC4ceO4+uqreeWVVwB8Ws/MrNbaqut3tbz66qtMmzaNq6++mkGDBnHxxRfzmc98hrvvvpvZs2czefJkXn31VQDOP/989ttvP2bNmsXxxx9P7969OfLII9mwYUPZZYwfP56lS5dSV1dHz549Oe6447jggguqvm7KenR3HHV1deGHDVpn0thTb9GiRTWNw+DJJ5/kgAMOqHUYHU5z203SIxFRt71t+rSemZkVjpOTmZkVjpOTmVlOrS91LF++nOXLl9c0htao1vZycjIzS3r16sX69etrnqA6isbnOfXq1avN23ZvPTOzZPDgwTQ0NLB27dqaxbBmzRoA3njjjZrF0BqNT8Jta05OZmZJjx492vyJrq31+c9/HnDvTZ/WMzOzwnFyMjOzwnFyMjOzwnFyMjOzwnFyMjOzwnFyMjOzwnFXcuvyhs64pabLX/P0+kLEUfQ7cFvX4iMnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHCcnMzMrHN9bz8wsp9b3OPS9FjM+cjIzs8JxcjIzs8JxcjIzs8JxcjIzs8KpanKSNF7Sckn1kmY0M30vSfdIelTSY5KOq2Y8ZmbWMVQtOUnaAbgMOBY4EJgs6cAm1f4DuD4i3gtMAi6vVjxmZtZxVPPI6VCgPiKejojNwFxgYpM6AbwjDe8CPFfFeMzMrIOo5u+c9gSezY03AIc1qXMucIekLwB9gKOrGI9ZIb3rE9+udQhmhVPrDhGTgdkRMRg4DviJpLfEJOk0SUskLVm7dm27B2lmZu2rmslpNTAkNz44leWdClwPEBG/AXoBA5o2FBGzIqIuIuoGDhxYpXDNzKwoqpmcFgPDJQ2T1JOsw8P8JnWeAY4CkHQAWXLyoZGZWRdXteQUEa8D04HbgSfJeuU9Iek8SRNSta8A0yT9DpgDTI2IqFZMZmbWMVT1xq8RsQBY0KTsnNzwMmBMNWMwM7OOp9YdIszMzN7CycnMzArHycnMzArHycnMzArHycnMzArHycnMzArHycnMzArHycnMzAqnqj/CNTOz1vFd6jM+cjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JxcjIzs8JpMTlJ6i3p65J+kMaHS/pw9UMzM7OuqpIjp6uBV4HRaXw1cH7VIjIzsy6vkuS0T0R8B3gNICJeAVTVqMzMrEurJDltlrQTEACS9iE7kmqRpPGSlkuqlzSjRJ2PSVom6QlJ11YcuZmZdVrdK6jzDeA2YIiknwFjgKktzSRpB+AyYBzQACyWND8iluXqDAf+DRgTES9K2r31q2BmZp1N2eQkqRuwG/BPwOFkp/O+GBHrKmj7UKA+Ip5Obc0FJgLLcnWmAZdFxIsAEfGXVq+BmZl1OmVP60XEG8BZEbE+Im6JiJsrTEwAewLP5sYbUlnefsB+kh6Q9KCk8c01JOk0SUskLVm7dm2Fizczs46qkmtOCyWdKWmIpH6NrzZafndgODAWmAz8QNKuTStFxKyIqIuIuoEDB7bRos3MrKgqueb08fT39FxZAO9uYb7VwJDc+OBUltcAPBQRrwErJa0gS1aLK4jLzMw6qRaTU0QM2862FwPDJQ0jS0qTgE80qfMLsiOmqyUNIDvN9/R2Ls/MzDqJFpOTpB7A54G/T0WLgCvT0U5JEfG6pOnA7cAOwFUR8YSk84AlETE/TTtG0jJgC/DViFi/3WtjZmadQiWn9f4H6AFcnsY/mco+29KMEbEAWNCk7JzccABfTi8zMzOgsuT0/og4ODd+t6TfVSsgMzOzSnrrbUl3hQBA0rvJTsGZmZlVRSVHTl8F7pH0NNmPcPcGTqlqVGZm1qVV0lvvrnSbof1T0fKIqOjeemZmZtujkuc5nQ7sFBGPRcRjQG9J/1L90MzMrKuq5JrTtIh4qXEk3QdvWvVCMjOzrq6S5LSDpK3Pb0p3G+9ZvZDMzKyrq6RDxG3AdZKuTOP/nMrMzMyqopLkdDZwGtldIgDuBH5YtYjMzKzLq6S33hvAFZKuAg4CVkeEf+dkZmZVU/Kak6QrJB2UhncBlgLXAI9KmtxO8ZmZWRdUrkPEkRHxRBo+BVgRESOB9wFnVT0yMzPrssolp8254XFkj7cgItZUNSIzM+vyyiWnlyR9WNJ7gTGkHnqSugM7tUdwZmbWNZXrEPHPwPeAdwFn5I6YjgJuqXZgZmbWdZVMThGxAhjfTPntZA8JNDMzq4pK7hBhZmbWrpyczMyscMr9zuld7RmImZlZo3JHTkslLZR0qqRd2y0iMzPr8solpz2BC4EjgOWSbpI0SZK7kZuZWVWVTE4RsSUibo+IU4AhwFXARGClpJ+1V4BmZtb1VNQhIiI2A8uAJ4G/AQdUMygzM+vayiYnSUMkfVXSb4GbU/0JETGqXaIzM7MuqeSPcCX9muy60/Vkj2p/pN2iMjOzLq3c7YtmAPdFRDQWSNoH+AQwKSIOqnZwZmbWNZXrEHFvRISkQZK+JGkx8ESaZ1K7RWhmZl1OuR/hnibpHmAR0B84FfhzRHwzIh5vp/jMzKwLKnda7/vAb4BPRMQSAElRpr6ZmVmbKJec9gBOAi5OtzK6HujRLlGZmVmXVu6a0/qIuCIiPkD2DKeXgOclPSnpgnaL0MzMupxKf4TbEBEXR0QdMAHYVN2wzMysKyvXIeJkSZ9sZtLhQH31QjIzs66u3JHTF4Abmym/AfhKdcIxMzMrn5x6RMTGpoUR8TLuGGFmZlVULjntJKlP00JJfYGe1QvJzMy6unLJ6Srg55L2biyQNBSYC/youmGZmVlXVi45TQZ+Adwrab2kF4BfATdHxIWVNC5pvKTlkuolzShT76OSQlJdq6I3M7NOqdyPcBURVwJXplN5RMSGShuWtANwGTAOaAAWS5ofEcua1OsLfBF4qLXBm5lZ51QuOQ2U9OV8gaStwxFxSQttHwrUR8TTad65ZE/SXdak3n8CM4GvVhizmZl1cuVO6+0A9C3zasmewLO58YZUtpWkUcCQiLilXEPpJrRLJC1Zu3ZtBYs2M7OOrNyR058j4pvVWrCkbsAlwNSW6kbELGAWQF1dnW8+a2bWyZU7clKZaZVYDQzJjQ9OZY36Au8BFklaRXbnifnuFGFmZuWS01Fvs+3FwHBJwyT1JHtA4fzGiRHx14gYEBFDI2Io8CAwofHxHGZm1nWVuyv5C2+n4Yh4HZgO3A48CVwfEU9IOk/ShLfTtpmZdW7lrjm9bRGxAFjQpOycEnXHVjMWMzPrOCp6ZIaZmVl7cnIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCcXIyM7PCqWpykjRe0nJJ9ZJmNDP9y5KWSXpM0l2S9q5mPGZm1jFULTlJ2gG4DDgWOBCYLOnAJtUeBeoi4u+AnwPfqVY8ZmbWcVTzyOlQoD4ino6IzcBcYGK+QkTcExGvpNEHgcFVjMfMzDqIaianPYFnc+MNqayUU4Fbm5sg6TRJSyQtWbt2bRuGaGZmRVSIDhGSTgbqgAubmx4RsyKiLiLqBg4c2L7BmZlZu+texbZXA0Ny44NT2TYkHQ38O/CBiHi1ivGYmVkHUc0jp8XAcEnDJPUEJgHz8xUkvRe4EpgQEX+pYixmZtaBVC05RcTrwHTgduBJ4PqIeELSeZImpGoXAjsD8yQtlTS/RHNmZtaFVPO0HhGxAFjQpOyc3PDR1Vy+mZl1TIXoEGFmZpbn5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5GRmZoXj5LSd/uvOFYz59t21DsPMrFNycmqF+/6wlu8uXMGGTa9tU37nsue58PanahSVmVnn073WAXQ0tz/xPLN/vYohu/Xmb5teY+JlD/D02o18ZswwIgJJtQ7RzKzDc3JqhSOHD2TBvw7gazf+njkPPwPAug2vsuBfj2RIv941js7MrPPwab1W+HX9Oj586f3c+vs/M3LPXejbqzsDdu7Jcd+7j/+6cwURUesQzcw6BSenVti85Q0+NGJ37j3rg3xoxO68o1cPbpp+BBeeeDCvvv6GT+mZmbURn9ZrhbH7787Y/Xd/S/n497yL8e95Vw0iMjPrnJycttOXxu3Hl8btV+swzMw6JZ/WMzOzwnFyMjOzwnFyMjOzwqlqcpI0XtJySfWSZjQzfUdJ16XpD0kaWs14zMysY6hahwhJOwCXAeOABmCxpPkRsSxX7VTgxYjYV9IkYCbw8WrF1GjojFuqvYgOYdW3j691CGZmzarmkdOhQH1EPB0Rm4G5wMQmdSYCP07DPweOkn8sZGbW5VWzK/mewLO58QbgsFJ1IuJ1SX8F+gPr8pUknQaclkY3SlpelYi7GM1kAE22tXVd3h8srw32h73fzvI7xO+cImIWMKvWcXQ2kpZERF2t47Bi8P5gebXeH6p5Wm81MCQ3PjiVNVtHUndgF2B9FWMyM7MOoJrJaTEwXNIwST2BScD8JnXmA59OwycCd4fvnmpm1uVV7bReuoY0Hbgd2AG4KiKekHQesCQi5gM/An4iqR54gSyBWfvxqVLL8/5geTXdH+QDFTMzKxrfIcLMzArHycnMzArHyakTkXSupDPLTJ8qaVBu/AxJrXq+vKSxkm5+O3Fa+2jp/ZX0Q0kHttGyNrZFO7b9JC2QtGut46hEJfuLk1PXMhUYlBs/A2hVcrIOpeT7K2mHiPhsk9uJWQcWEcdFxEu1jqOtODl1cJL+XdIKSfcD+6eyQyQ9KOkxSTdK2k3SiUAd8DNJSyV9kSxR3SPpnjTfMZJ+I+m3kuZJ2jmVj5f0lKTfAv9UmzW1ciT1kXSLpN9J+r2kb/DW93ejpIsl/Q4YLWmRpLrctP+S9ISkuyQNTOX7SLpN0iOS7pM0IpUPS/vK45LOr9Fqd1mSPpf+j5dKWinpHkmrJA1I07+ebrp9v6Q5ks6UNCg3z1JJWyTtLekj6cbbj0paKOmdqY2Bku5M+8QPJf2pXPupvO32l4jwq4O+gPcBj5N9O34HUA+cCTwGfCDVOQ/4bhpeBNTl5l8FDEjDA4B7gT5p/GzgHKAX2S2mhgMCrgdurvW6+/WWfeGjwA9y47vk399UFsDHcuNb94c0bUoaPgf4fhq+Cxiehg8j+y0iZL9R/FQaPh3YWOtt0BVfQA/gPuAjje838H5gafrf7Qv8ATizyXynA9en4d14s+f2Z4GL0/D3gX9Lw+PTPlK2/bbcXzrE7YuspCOBGyPiFQBJ84E+wK4R8atU58fAvAraOhw4EHgg3Xu3J/AbYASwMiL+kJbxU968z6EVx+PAxZJmkn15uK+ZeyhvAf63xPxvANel4Z8CN6Qj5/8HzMu1tWP6O4YsIQL8hOyJAtb+/pssAfxS0qWpbAxwU0RsAjZJ+mV+BkljgGnAEaloMHCdpD3I/u9XpvIjgH8EiIjbJL1Yrv223l+cnKyRgDsjYvI2hdIhNYrHWiEiVkgaBRwHnC/prmaqbYqILZU2SXba/6WIKLUP+EeSNSRpKtnNVae3Yp49yG5+MCEiGjslXApcEhHzJY0Fzt3OkNp0f/E1p47tXuAESTtJ6kt2aP8y8KKkI1OdTwKNR1EbyA7DaWb8QWCMpH1h6zWM/YCngKGS9kn1tkleVgzKemG+EhE/BS4ERvHW97ucbmS3EAP4BHB/RPwNWCnppLQMSTo41SEtK5wAAAF0SURBVHmAN+/oMqUNVsFaQdL7yE7hnxwRbzSZ/ADwEUm90tHMh9M8PcjOopwdESty9XfhzfuefrpJOx9L8x5DdvqvZPttvb84OXVgEfFbslMxvwNuJbufIWQ72IWSHgMOIbvuBDAbuCJdDN2J7PYkt0m6JyLWkvXmm5Pm+w0wIh26nwbckjpE/KVdVs5aayTwsKSlwDeA88m9vxXM/zJwqKTfAx/izX1mCnBq6kTxBG8+k+2LwOmSHid79I21r+lAP7IOL0sl/bBxQkQsJrvG8xjZ58LjwF/JTrnVAd/MdYoYRHakNE/SI2z7iIxvAsekfeIkYA2woUz70Ib7i29fZGZI2hgRO9c6DmsbknaOiI3Kfud2L3Ba+jLbmjZ2BLZEdp/U0cD/NJ6ya4v2W+JrTmZmnc8sZT+w7gX8eDsTx17A9ZK6AZvJOlG0Zftl+cjJzMwKx9eczMyscJyczMyscJyczMyscJyczMyscJyczMyscP4/+O6kk1+L5bsAAAAASUVORK5CYII=\n" - }, - "metadata": { - "tags": [], - "needs_background": "light" - } + "ename": "KeyError", + "evalue": "'mixed3a'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/bx/1rzxld1d3pdfks1bqj8dn3jw0000gn/T/ipykernel_11873/2088044462.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mutils_plot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_random_exp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_random_exp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/MastersThesis/tcav/tcav/utils_plot.py\u001b[0m in \u001b[0;36mplot_results\u001b[0;34m(results, random_counterpart, random_concepts, num_random_exp, min_p_val)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;31m# Calculate statistical significance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mttest_ind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_i_ups\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbottleneck\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi_ups\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mbottleneck\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mplot_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyError\u001b[0m: 'mixed3a'" + ] } + ], + "source": [ + "utils_plot.plot_results(results, num_random_exp=num_random_exp)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "name": "Run TCAV.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.9.2 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.2" + }, + "vscode": { + "interpreter": { + "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/requirements.txt b/requirements.txt index d6ce474847..6b045be80c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ +# first: +# pip install tensorflow +# pip intsall tcav +# then you should not need this file matplotlib==2.2.4 Pillow==8.2.0 scikit-learn==0.20.3 scipy==1.2.1 -tensorflow==2.5.1 +tensorflow==2.5.0 numpy==1.19.2 protobuf==3.10.0 pandas==1.0.3 diff --git a/tcav/cav.py b/tcav/cav.py index 8498ee5a48..66f79e4c11 100644 --- a/tcav/cav.py +++ b/tcav/cav.py @@ -29,293 +29,330 @@ class CAV(object): - """CAV class contains methods for concept activation vector (CAV). + """CAV class contains methods for concept activation vector (CAV). - CAV represents semenatically meaningful vector directions in - network's embeddings (bottlenecks). - """ - - @staticmethod - def default_hparams(): - """HParams used to train the CAV. - - you can use logistic regression or linear regression, or different - regularization of the CAV parameters. - - Returns: - TF.HParams for training. - """ - return {'model_type':'linear', 'alpha':.01, 'max_iter':1000, 'tol':1e-3} - - @staticmethod - def load_cav(cav_path): - """Make a CAV instance from a saved CAV (pickle file). - - Args: - cav_path: the location of the saved CAV - - Returns: - CAV instance. + CAV represents semenatically meaningful vector directions in + network's embeddings (bottlenecks). """ - with tf.io.gfile.GFile(cav_path, 'rb') as pkl_file: - save_dict = pickle.load(pkl_file) - - cav = CAV(save_dict['concepts'], save_dict['bottleneck'], - save_dict['hparams'], save_dict['saved_path']) - cav.accuracies = save_dict['accuracies'] - cav.cavs = save_dict['cavs'] - return cav - - @staticmethod - def cav_key(concepts, bottleneck, model_type, alpha): - """A key of this cav (useful for saving files). - - Args: - concepts: set of concepts used for CAV - bottleneck: the bottleneck used for CAV - model_type: the name of model for CAV - alpha: a parameter used to learn CAV - Returns: - a string cav_key + @staticmethod + def default_hparams(): + """HParams used to train the CAV. + + you can use logistic regression or linear regression, or different + regularization of the CAV parameters. + + Returns: + TF.HParams for training. + """ + return {"model_type": "linear", "alpha": 0.01, "max_iter": 1000, "tol": 1e-3} + + @staticmethod + def load_cav(cav_path): + """Make a CAV instance from a saved CAV (pickle file). + + Args: + cav_path: the location of the saved CAV + + Returns: + CAV instance. + """ + with tf.io.gfile.GFile(cav_path, "rb") as pkl_file: + save_dict = pickle.load(pkl_file) + + cav = CAV( + save_dict["concepts"], + save_dict["bottleneck"], + save_dict["hparams"], + save_dict["saved_path"], + ) + cav.accuracies = save_dict["accuracies"] + cav.cavs = save_dict["cavs"] + return cav + + @staticmethod + def cav_key(concepts, bottleneck, model_type, alpha): + """A key of this cav (useful for saving files). + + Args: + concepts: set of concepts used for CAV + bottleneck: the bottleneck used for CAV + model_type: the name of model for CAV + alpha: a parameter used to learn CAV + + Returns: + a string cav_key + """ + return ( + "-".join([str(c) for c in concepts]) + + "-" + + bottleneck + + "-" + + model_type + + "-" + + str(alpha) + ) + + @staticmethod + def check_cav_exists(cav_dir, concepts, bottleneck, cav_hparams): + """Check if a CAV is saved in cav_dir. + + Args: + cav_dir: where cav pickles might be saved + concepts: set of concepts used for CAV + bottleneck: the bottleneck used for CAV + cav_hparams: a parameter used to learn CAV + + Returns: + True if exists, False otherwise. + """ + cav_path = os.path.join( + cav_dir, + CAV.cav_key( + concepts, bottleneck, cav_hparams["model_type"], cav_hparams["alpha"] + ) + + ".pkl", + ) + return tf.io.gfile.exists(cav_path) + + @staticmethod + def _create_cav_training_set(concepts, bottleneck, acts): + """Flattens acts, make mock-labels and returns the info. + + Labels are assigned in the order that concepts exists. + + Args: + concepts: names of concepts + bottleneck: the name of bottleneck where acts come from + acts: a dictionary that contains activations + Returns: + x - flattened acts + labels - corresponding labels (integer) + labels2text - map between labels and text. + """ + + x = [] + labels = [] + labels2text = {} + # to make sure postiive and negative examples are balanced, + # truncate all examples to the size of the smallest concept. + min_data_points = np.min( + [acts[concept][bottleneck].shape[0] for concept in acts.keys()] + ) + + for i, concept in enumerate(concepts): + x.extend( + acts[concept][bottleneck][:min_data_points].reshape(min_data_points, -1) + ) + + labels.extend([i] * min_data_points) + labels2text[i] = concept + x = np.array(x) + labels = np.array(labels) + + return x, labels, labels2text + + def __init__(self, concepts, bottleneck, hparams, save_path=None): + """Initialize CAV class. + + Args: + concepts: set of concepts used for CAV + bottleneck: the bottleneck used for CAV + hparams: a parameter used to learn CAV + save_path: where to save this CAV + """ + self.concepts = concepts + self.bottleneck = bottleneck + self.hparams = hparams + self.save_path = save_path + + def train(self, acts): + """Train the CAVs from the activations. + + Args: + acts: is a dictionary of activations. In particular, acts takes for of + {'concept1':{'bottleneck name1':[...act array...], + 'bottleneck name2':[...act array...],... + 'concept2':{'bottleneck name1':[...act array...], + Raises: + ValueError: if the model_type in hparam is not compatible. + """ + + tf.compat.v1.logging.info( + "training with alpha={}".format(self.hparams["alpha"]) + ) + x, labels, labels2text = CAV._create_cav_training_set( + self.concepts, self.bottleneck, acts + ) + + if self.hparams["model_type"] == "linear": + lm = linear_model.SGDClassifier( + alpha=self.hparams["alpha"], + max_iter=self.hparams["max_iter"], + tol=self.hparams["tol"], + ) + elif self.hparams["model_type"] == "logistic": + lm = linear_model.LogisticRegression() + else: + raise ValueError( + "Invalid hparams.model_type: {}".format(self.hparams["model_type"]) + ) + + self.accuracies = self._train_lm(lm, x, labels, labels2text) + if len(lm.coef_) == 1: + # if there were only two labels, the concept is assigned to label 0 by + # default. So we flip the coef_ to reflect this. + self.cavs = [-1 * lm.coef_[0], lm.coef_[0]] + else: + self.cavs = [c for c in lm.coef_] + self._save_cavs() + + def perturb_act(self, act, concept, operation=np.add, alpha=1.0): + """Make a perturbation of act with a direction of this CAV. + + Args: + act: activations to be perturbed + concept: the concept to perturb act with. + operation: the operation will be ran to perturb. + alpha: size of the step. + + Returns: + perturbed activation: same shape as act + """ + flat_act = np.reshape(act, -1) + pert = operation(flat_act, alpha * self.get_direction(concept)) + return np.reshape(pert, act.shape) + + def get_key(self): + """Returns cav_key.""" + + return CAV.cav_key( + self.concepts, + self.bottleneck, + self.hparams["model_type"], + self.hparams["alpha"], + ) + + def get_direction(self, concept): + """Get CAV direction. + + Args: + concept: the conept of interest + + Returns: + CAV vector. + """ + return self.cavs[self.concepts.index(concept)] + + def _save_cavs(self): + """Save a dictionary of this CAV to a pickle.""" + save_dict = { + "concepts": self.concepts, + "bottleneck": self.bottleneck, + "hparams": self.hparams, + "accuracies": self.accuracies, + "cavs": self.cavs, + "saved_path": self.save_path, + } + if self.save_path is not None: + with tf.io.gfile.GFile(self.save_path, "w") as pkl_file: + pickle.dump(save_dict, pkl_file) + else: + tf.compat.v1.logging.info("save_path is None. Not saving anything") + + def _train_lm(self, lm, x, y, labels2text): + """Train a model to get CAVs. + + Modifies lm by calling the lm.fit functions. The cav coefficients are then + in lm._coefs. + + Args: + lm: An sklearn linear_model object. Can be linear regression or + logistic regression. Must support .fit and ._coef. + x: An array of training data of shape [num_data, data_dim] + y: An array of integer labels of shape [num_data] + labels2text: Dictionary of text for each label. + + Returns: + Dictionary of accuracies of the CAVs. + + """ + x_train, x_test, y_train, y_test = train_test_split( + x, y, test_size=0.33, stratify=y + ) + # if you get setting an array element with a sequence, chances are that your + # each of your activation had different shape - make sure they are all from + # the same layer, and input image size was the same + lm.fit(x_train, y_train) + y_pred = lm.predict(x_test) + # get acc for each class. + num_classes = max(y) + 1 + acc = {} + num_correct = 0 + for class_id in range(num_classes): + # get indices of all test data that has this class. + idx = y_test == class_id + acc[labels2text[class_id]] = metrics.accuracy_score( + y_pred[idx], y_test[idx] + ) + # overall correctness is weighted by the number of examples in this class. + num_correct += sum(idx) * acc[labels2text[class_id]] + acc["overall"] = float(num_correct) / float(len(y_test)) + tf.compat.v1.logging.info("acc per class %s" % (str(acc))) + return acc + + +def get_or_train_cav( + concepts, bottleneck, acts, cav_dir=None, cav_hparams=None, overwrite=False +): """ - return '-'.join([str(c) for c in concepts - ]) + '-' + bottleneck + '-' + model_type + '-' + str(alpha) + Gets, creating and training if necessary, the specified CAV. - @staticmethod - def check_cav_exists(cav_dir, concepts, bottleneck, cav_hparams): - """Check if a CAV is saved in cav_dir. + Assumes the activations already exists. Args: - cav_dir: where cav pickles might be saved concepts: set of concepts used for CAV + Note: if there are two concepts, provide the positive concept + first, then negative concept (e.g., ['striped', 'random500_1'] bottleneck: the bottleneck used for CAV + acts: dictionary contains activations of concepts in each bottlenecks + e.g., acts[concept][bottleneck] + cav_dir: a directory to store the results. cav_hparams: a parameter used to learn CAV + overwrite: if set to True overwrite any saved CAV files. Returns: - True if exists, False otherwise. - """ - cav_path = os.path.join( - cav_dir, - CAV.cav_key(concepts, bottleneck, cav_hparams['model_type'], - cav_hparams['alpha']) + '.pkl') - return tf.io.gfile.exists(cav_path) - - @staticmethod - def _create_cav_training_set(concepts, bottleneck, acts): - """Flattens acts, make mock-labels and returns the info. - - Labels are assigned in the order that concepts exists. - - Args: - concepts: names of concepts - bottleneck: the name of bottleneck where acts come from - acts: a dictionary that contains activations - Returns: - x - flattened acts - labels - corresponding labels (integer) - labels2text - map between labels and text. + returns a CAV instance """ - x = [] - labels = [] - labels2text = {} - # to make sure postiive and negative examples are balanced, - # truncate all examples to the size of the smallest concept. - min_data_points = np.min( - [acts[concept][bottleneck].shape[0] for concept in acts.keys()]) - - for i, concept in enumerate(concepts): - x.extend(acts[concept][bottleneck][:min_data_points].reshape( - min_data_points, -1)) - labels.extend([i] * min_data_points) - labels2text[i] = concept - x = np.array(x) - labels = np.array(labels) - - return x, labels, labels2text - - def __init__(self, concepts, bottleneck, hparams, save_path=None): - """Initialize CAV class. - - Args: - concepts: set of concepts used for CAV - bottleneck: the bottleneck used for CAV - hparams: a parameter used to learn CAV - save_path: where to save this CAV - """ - self.concepts = concepts - self.bottleneck = bottleneck - self.hparams = hparams - self.save_path = save_path - - def train(self, acts): - """Train the CAVs from the activations. - - Args: - acts: is a dictionary of activations. In particular, acts takes for of - {'concept1':{'bottleneck name1':[...act array...], - 'bottleneck name2':[...act array...],... - 'concept2':{'bottleneck name1':[...act array...], - Raises: - ValueError: if the model_type in hparam is not compatible. - """ - - tf.compat.v1.logging.info('training with alpha={}'.format(self.hparams['alpha'])) - x, labels, labels2text = CAV._create_cav_training_set( - self.concepts, self.bottleneck, acts) - - if self.hparams['model_type'] == 'linear': - lm = linear_model.SGDClassifier(alpha=self.hparams['alpha'], max_iter=self.hparams['max_iter'], tol=self.hparams['tol']) - elif self.hparams['model_type'] == 'logistic': - lm = linear_model.LogisticRegression() - else: - raise ValueError('Invalid hparams.model_type: {}'.format( - self.hparams['model_type'])) - - self.accuracies = self._train_lm(lm, x, labels, labels2text) - if len(lm.coef_) == 1: - # if there were only two labels, the concept is assigned to label 0 by - # default. So we flip the coef_ to reflect this. - self.cavs = [-1 * lm.coef_[0], lm.coef_[0]] - else: - self.cavs = [c for c in lm.coef_] - self._save_cavs() - - def perturb_act(self, act, concept, operation=np.add, alpha=1.0): - """Make a perturbation of act with a direction of this CAV. - - Args: - act: activations to be perturbed - concept: the concept to perturb act with. - operation: the operation will be ran to perturb. - alpha: size of the step. - - Returns: - perturbed activation: same shape as act - """ - flat_act = np.reshape(act, -1) - pert = operation(flat_act, alpha * self.get_direction(concept)) - return np.reshape(pert, act.shape) - - def get_key(self): - """Returns cav_key.""" - - return CAV.cav_key(self.concepts, self.bottleneck, self.hparams['model_type'], - self.hparams['alpha']) - - def get_direction(self, concept): - """Get CAV direction. - - Args: - concept: the conept of interest - - Returns: - CAV vector. - """ - return self.cavs[self.concepts.index(concept)] - - def _save_cavs(self): - """Save a dictionary of this CAV to a pickle.""" - save_dict = { - 'concepts': self.concepts, - 'bottleneck': self.bottleneck, - 'hparams': self.hparams, - 'accuracies': self.accuracies, - 'cavs': self.cavs, - 'saved_path': self.save_path - } - if self.save_path is not None: - with tf.io.gfile.GFile(self.save_path, 'w') as pkl_file: - pickle.dump(save_dict, pkl_file) - else: - tf.compat.v1.logging.info('save_path is None. Not saving anything') - - def _train_lm(self, lm, x, y, labels2text): - """Train a model to get CAVs. - - Modifies lm by calling the lm.fit functions. The cav coefficients are then - in lm._coefs. - - Args: - lm: An sklearn linear_model object. Can be linear regression or - logistic regression. Must support .fit and ._coef. - x: An array of training data of shape [num_data, data_dim] - y: An array of integer labels of shape [num_data] - labels2text: Dictionary of text for each label. - - Returns: - Dictionary of accuracies of the CAVs. - - """ - x_train, x_test, y_train, y_test = train_test_split( - x, y, test_size=0.33, stratify=y) - # if you get setting an array element with a sequence, chances are that your - # each of your activation had different shape - make sure they are all from - # the same layer, and input image size was the same - lm.fit(x_train, y_train) - y_pred = lm.predict(x_test) - # get acc for each class. - num_classes = max(y) + 1 - acc = {} - num_correct = 0 - for class_id in range(num_classes): - # get indices of all test data that has this class. - idx = (y_test == class_id) - acc[labels2text[class_id]] = metrics.accuracy_score( - y_pred[idx], y_test[idx]) - # overall correctness is weighted by the number of examples in this class. - num_correct += (sum(idx) * acc[labels2text[class_id]]) - acc['overall'] = float(num_correct) / float(len(y_test)) - tf.compat.v1.logging.info('acc per class %s' % (str(acc))) - return acc - - -def get_or_train_cav(concepts, - bottleneck, - acts, - cav_dir=None, - cav_hparams=None, - overwrite=False): - """Gets, creating and training if necessary, the specified CAV. - - Assumes the activations already exists. - - Args: - concepts: set of concepts used for CAV - Note: if there are two concepts, provide the positive concept - first, then negative concept (e.g., ['striped', 'random500_1'] - bottleneck: the bottleneck used for CAV - acts: dictionary contains activations of concepts in each bottlenecks - e.g., acts[concept][bottleneck] - cav_dir: a directory to store the results. - cav_hparams: a parameter used to learn CAV - overwrite: if set to True overwrite any saved CAV files. - - Returns: - returns a CAV instance - """ - - if cav_hparams is None: - cav_hparams = CAV.default_hparams() - - cav_path = None - if cav_dir is not None: - utils.make_dir_if_not_exists(cav_dir) - cav_path = os.path.join( - cav_dir, - CAV.cav_key(concepts, bottleneck, cav_hparams['model_type'], - cav_hparams['alpha']).replace('/', '.') + '.pkl') - - if not overwrite and tf.io.gfile.exists(cav_path): - tf.compat.v1.logging.info('CAV already exists: {}'.format(cav_path)) - cav_instance = CAV.load_cav(cav_path) - tf.compat.v1.logging.info('CAV accuracies: {}'.format(cav_instance.accuracies)) - return cav_instance - - tf.compat.v1.logging.info('Training CAV {} - {} alpha {}'.format( - concepts, bottleneck, cav_hparams['alpha'])) - cav_instance = CAV(concepts, bottleneck, cav_hparams, cav_path) - cav_instance.train({c: acts[c] for c in concepts}) - tf.compat.v1.logging.info('CAV accuracies: {}'.format(cav_instance.accuracies)) - return cav_instance + if cav_hparams is None: + cav_hparams = CAV.default_hparams() + + cav_path = None + if cav_dir is not None: + utils.make_dir_if_not_exists(cav_dir) + cav_path = os.path.join( + cav_dir, + CAV.cav_key( + concepts, bottleneck, cav_hparams["model_type"], cav_hparams["alpha"] + ).replace("/", ".") + + ".pkl", + ) + + if not overwrite and tf.io.gfile.exists(cav_path): + tf.compat.v1.logging.info("CAV already exists: {}".format(cav_path)) + cav_instance = CAV.load_cav(cav_path) + tf.compat.v1.logging.info( + "CAV accuracies: {}".format(cav_instance.accuracies) + ) + return cav_instance + + tf.compat.v1.logging.info( + "Training CAV {} - {} alpha {}".format( + concepts, bottleneck, cav_hparams["alpha"] + ) + ) + cav_instance = CAV(concepts, bottleneck, cav_hparams, cav_path) + cav_instance.train({c: acts[c] for c in concepts}) + tf.compat.v1.logging.info("CAV accuracies: {}".format(cav_instance.accuracies)) + return cav_instance diff --git a/tcav/model.py b/tcav/model.py index c57b21e5c4..07199eefba 100644 --- a/tcav/model.py +++ b/tcav/model.py @@ -24,399 +24,421 @@ import tensorflow as tf from google.protobuf import text_format + class ModelWrapper(six.with_metaclass(ABCMeta, object)): - """Simple wrapper of the for models with session object for TCAV. + """Simple wrapper of the for models with session object for TCAV. Supports easy inference with no need to deal with the feed_dicts. - """ - - @abstractmethod - def __init__(self, model_path=None, node_dict=None): - """Initialize the wrapper. - - Optionally create a session, load - the model from model_path to this session, and map the - input/output and bottleneck tensors. - - Args: - model_path: one of the following: 1) Directory path to checkpoint 2) - Directory path to SavedModel 3) File path to frozen graph.pb 4) File - path to frozen graph.pbtxt - node_dict: mapping from a short name to full input/output and bottleneck - tensor names. Users should pass 'input' and 'prediction' - as keys and the corresponding input and prediction tensor - names as values in node_dict. Users can additionally pass bottleneck - tensor names for which gradient Ops will be added later. - """ - # A dictionary of bottleneck tensors. - self.bottlenecks_tensors = None - # A dictionary of input, 'logit' and prediction tensors. - self.ends = None - # The model name string. - self.model_name = None - # a place holder for index of the neuron/class of interest. - # usually defined under the graph. For example: - # with g.as_default(): - # self.tf.placeholder(tf.int64, shape=[None]) - self.y_input = None - # The tensor representing the loss (used to calculate derivative). - self.loss = None - # If tensors in the loaded graph are prefixed with 'import/' - self.import_prefix = False - - if model_path: - self._try_loading_model(model_path) - if node_dict: - self._find_ends_and_bottleneck_tensors(node_dict) - - def _try_loading_model(self, model_path): - """ Load model from model_path. - - TF models are often saved in one of the three major formats: - 1) Checkpoints with ckpt.meta, ckpt.data, and ckpt.index. - 2) SavedModel format with saved_model.pb and variables/. - 3) Frozen graph in .pb or .pbtxt format. - When model_path is specified, model is loaded in one of the - three formats depending on the model_path. When model_path is - ommitted, child wrapper is responsible for loading the model. - """ - try: - self.sess = tf.compat.v1.Session(graph=tf.Graph()) - with self.sess.graph.as_default(): - if tf.io.gfile.isdir(model_path): - ckpt = tf.train.latest_checkpoint(model_path) - if ckpt: - tf.compat.v1.logging.info('Loading from the latest checkpoint.') - saver = tf.compat.v1.train.import_meta_graph(ckpt + '.meta') - saver.restore(self.sess, ckpt) - else: - tf.compat.v1.logging.info('Loading from SavedModel dir.') - tf.compat.v1.saved_model.loader.load(self.sess, ['serve'], model_path) - else: - input_graph_def = tf.compat.v1.GraphDef() - if model_path.endswith('.pb'): - tf.compat.v1.logging.info('Loading from frozen binary graph.') - with tf.io.gfile.GFile(model_path, 'rb') as f: - input_graph_def.ParseFromString(f.read()) - else: - tf.compat.v1.logging.info('Loading from frozen text graph.') - with tf.io.gfile.GFile(model_path) as f: - text_format.Parse(f.read(), input_graph_def) - tf.import_graph_def(input_graph_def) - self.import_prefix = True - - except Exception as e: - template = 'An exception of type {0} occurred ' \ - 'when trying to load model from {1}. ' \ - 'Arguments:\n{2!r}' - tf.compat.v1.logging.warn(template.format(type(e).__name__, model_path, e.args)) - - def _find_ends_and_bottleneck_tensors(self, node_dict): - """ Find tensors from the graph by their names. - - Depending on how the model is loaded, tensors in the graph - may or may not have 'import/' prefix added to every tensor name. - This is true even if the tensors already have 'import/' prefix. - The 'ends' and 'bottlenecks_tensors' dictionary should map to tensors - with the according name. - """ - self.bottlenecks_tensors = {} - self.ends = {} - for k, v in six.iteritems(node_dict): - if self.import_prefix: - v = 'import/' + v - tensor = self.sess.graph.get_operation_by_name(v.strip(':0')).outputs[0] - if k == 'input' or k == 'prediction': - self.ends[k] = tensor - else: - self.bottlenecks_tensors[k] = tensor - - def _make_gradient_tensors(self): - """Makes gradient tensors for all bottleneck tensors.""" - - self.bottlenecks_gradients = {} - for bn in self.bottlenecks_tensors: - self.bottlenecks_gradients[bn] = tf.gradients( - ys=self.loss, xs=self.bottlenecks_tensors[bn])[0] - - def get_gradient(self, acts, y, bottleneck_name, example): - """Return the gradient of the loss with respect to the bottleneck_name. - - Args: - acts: activation of the bottleneck - y: index of the logit layer - bottleneck_name: name of the bottleneck to get gradient wrt. - example: input example. Unused by default. Necessary for getting gradients - from certain models, such as BERT. - - Returns: - the gradient array. - """ - return self.sess.run(self.bottlenecks_gradients[bottleneck_name], { - self.bottlenecks_tensors[bottleneck_name]: acts, - self.y_input: y - }) - - def get_predictions(self, examples): - """Get prediction of the examples. - - Args: - imgs: array of examples to get predictions - - Returns: - array of predictions - """ - return self.adjust_prediction( - self.sess.run(self.ends['prediction'], {self.ends['input']: examples})) - - def adjust_prediction(self, pred_t): - """Adjust the prediction tensor to be the expected shape. - - Defaults to a no-op, but necessary to override for GoogleNet - Returns: - pred_t: pred_tensor. - """ - return pred_t - - def reshape_activations(self, layer_acts): - """Reshapes layer activations as needed to feed through the model network. - - Override this for models that require reshaping of the activations for use - in TCAV. - - Args: - layer_acts: Activations as returned by run_examples. - - Returns: - Activations in model-dependent form; the default is a squeezed array (i.e. - at most one dimensions of size 1). """ - return np.asarray(layer_acts).squeeze() - def label_to_id(self, label): - """Convert label (string) to index in the logit layer (id). - - Override this method if label to id mapping is known. Otherwise, - default id 0 is used. - """ - tf.compat.v1.logging.warn('label_to_id undefined. Defaults to returning 0.') - return 0 - - - def id_to_label(self, idx): - """Convert index in the logit layer (id) to label (string). - - Override this method if id to label mapping is known. - """ - return str(idx) - - def run_examples(self, examples, bottleneck_name): - """Get activations at a bottleneck for provided examples. - - Args: - examples: example data to feed into network. - bottleneck_name: string, should be key of self.bottlenecks_tensors - - Returns: - Activations in the given layer. - """ - return self.sess.run(self.bottlenecks_tensors[bottleneck_name], - {self.ends['input']: examples}) + @abstractmethod + def __init__(self, model_path=None, node_dict=None): + """Initialize the wrapper. + + Optionally create a session, load + the model from model_path to this session, and map the + input/output and bottleneck tensors. + + Args: + model_path: one of the following: 1) Directory path to checkpoint 2) + Directory path to SavedModel 3) File path to frozen graph.pb 4) File + path to frozen graph.pbtxt + node_dict: mapping from a short name to full input/output and bottleneck + tensor names. Users should pass 'input' and 'prediction' + as keys and the corresponding input and prediction tensor + names as values in node_dict. Users can additionally pass bottleneck + tensor names for which gradient Ops will be added later. + """ + # A dictionary of bottleneck tensors. + self.bottlenecks_tensors = None + # A dictionary of input, 'logit' and prediction tensors. + self.ends = None + # The model name string. + self.model_name = None + # a place holder for index of the neuron/class of interest. + # usually defined under the graph. For example: + # with g.as_default(): + # self.tf.placeholder(tf.int64, shape=[None]) + self.y_input = None + # The tensor representing the loss (used to calculate derivative). + self.loss = None + # If tensors in the loaded graph are prefixed with 'import/' + self.import_prefix = False + + if model_path: + self._try_loading_model(model_path) + if node_dict: + self._find_ends_and_bottleneck_tensors(node_dict) + + def _try_loading_model(self, model_path): + """Load model from model_path. + + TF models are often saved in one of the three major formats: + 1) Checkpoints with ckpt.meta, ckpt.data, and ckpt.index. + 2) SavedModel format with saved_model.pb and variables/. + 3) Frozen graph in .pb or .pbtxt format. + When model_path is specified, model is loaded in one of the + three formats depending on the model_path. When model_path is + ommitted, child wrapper is responsible for loading the model. + """ + try: + self.sess = tf.compat.v1.Session(graph=tf.Graph()) + with self.sess.graph.as_default(): + if tf.io.gfile.isdir(model_path): + ckpt = tf.train.latest_checkpoint(model_path) + if ckpt: + tf.compat.v1.logging.info("Loading from the latest checkpoint.") + saver = tf.compat.v1.train.import_meta_graph(ckpt + ".meta") + saver.restore(self.sess, ckpt) + else: + tf.compat.v1.logging.info("Loading from SavedModel dir.") + tf.compat.v1.saved_model.loader.load( + self.sess, ["serve"], model_path + ) + else: + input_graph_def = tf.compat.v1.GraphDef() + if model_path.endswith(".pb"): + tf.compat.v1.logging.info("Loading from frozen binary graph.") + with tf.io.gfile.GFile(model_path, "rb") as f: + input_graph_def.ParseFromString(f.read()) + else: + tf.compat.v1.logging.info("Loading from frozen text graph.") + with tf.io.gfile.GFile(model_path) as f: + text_format.Parse(f.read(), input_graph_def) + tf.import_graph_def(input_graph_def) + self.import_prefix = True + + except Exception as e: + template = ( + "An exception of type {0} occurred " + "when trying to load model from {1}. " + "Arguments:\n{2!r}" + ) + tf.compat.v1.logging.warn( + template.format(type(e).__name__, model_path, e.args) + ) + + def _find_ends_and_bottleneck_tensors(self, node_dict): + """Find tensors from the graph by their names. + + Depending on how the model is loaded, tensors in the graph + may or may not have 'import/' prefix added to every tensor name. + This is true even if the tensors already have 'import/' prefix. + The 'ends' and 'bottlenecks_tensors' dictionary should map to tensors + with the according name. + """ + self.bottlenecks_tensors = {} + self.ends = {} + for k, v in six.iteritems(node_dict): + if self.import_prefix: + v = "import/" + v + tensor = self.sess.graph.get_operation_by_name(v.strip(":0")).outputs[0] + if k == "input" or k == "prediction": + self.ends[k] = tensor + else: + self.bottlenecks_tensors[k] = tensor + + def _make_gradient_tensors(self): + """Makes gradient tensors for all bottleneck tensors.""" + + self.bottlenecks_gradients = {} + for bn in self.bottlenecks_tensors: + self.bottlenecks_gradients[bn] = tf.gradients( + ys=self.loss, xs=self.bottlenecks_tensors[bn] + )[0] + + def get_gradient(self, acts, y, bottleneck_name, example): + """Return the gradient of the loss with respect to the bottleneck_name. + + Args: + acts: activation of the bottleneck + y: index of the logit layer + bottleneck_name: name of the bottleneck to get gradient wrt. + example: input example. Unused by default. Necessary for getting gradients + from certain models, such as BERT. + + Returns: + the gradient array. + """ + return self.sess.run( + self.bottlenecks_gradients[bottleneck_name], + {self.bottlenecks_tensors[bottleneck_name]: acts, self.y_input: y}, + ) + + def get_predictions(self, examples): + """Get prediction of the examples. + + Args: + imgs: array of examples to get predictions + + Returns: + array of predictions + """ + return self.adjust_prediction( + self.sess.run(self.ends["prediction"], {self.ends["input"]: examples}) + ) + + def adjust_prediction(self, pred_t): + """Adjust the prediction tensor to be the expected shape. + + Defaults to a no-op, but necessary to override for GoogleNet + Returns: + pred_t: pred_tensor. + """ + return pred_t + + def reshape_activations(self, layer_acts): + """Reshapes layer activations as needed to feed through the model network. + + Override this for models that require reshaping of the activations for use + in TCAV. + + Args: + layer_acts: Activations as returned by run_examples. + + Returns: + Activations in model-dependent form; the default is a squeezed array (i.e. + at most one dimensions of size 1). + """ + return np.asarray(layer_acts).squeeze() + + def label_to_id(self, label): + """Convert label (string) to index in the logit layer (id). + + Override this method if label to id mapping is known. Otherwise, + default id 0 is used. + """ + tf.compat.v1.logging.warn("label_to_id undefined. Defaults to returning 0.") + return 0 + + def id_to_label(self, idx): + """Convert index in the logit layer (id) to label (string). + + Override this method if id to label mapping is known. + """ + return str(idx) + + def run_examples(self, examples, bottleneck_name): + """Get activations at a bottleneck for provided examples. + + Args: + examples: example data to feed into network. + bottleneck_name: string, should be key of self.bottlenecks_tensors + + Returns: + Activations in the given layer. + """ + return self.sess.run( + self.bottlenecks_tensors[bottleneck_name], {self.ends["input"]: examples} + ) class ImageModelWrapper(ModelWrapper): - """Wrapper base class for image models.""" + """Wrapper base class for image models.""" - def __init__(self, image_shape): - super(ModelWrapper, self).__init__() - # shape of the input image in this model - self.image_shape = image_shape + def __init__(self, image_shape): + super(ModelWrapper, self).__init__() + # shape of the input image in this model + self.image_shape = image_shape - def get_image_shape(self): - """returns the shape of an input image.""" - return self.image_shape + def get_image_shape(self): + """returns the shape of an input image.""" + return self.image_shape class PublicImageModelWrapper(ImageModelWrapper): - """Simple wrapper of the public image models with session object.""" - - def __init__(self, sess, model_fn_path, labels_path, image_shape, - endpoints_dict, scope): - super(PublicImageModelWrapper, self).__init__(image_shape) - self.labels = tf.io.gfile.GFile(labels_path).read().splitlines() - self.ends = PublicImageModelWrapper.import_graph( - model_fn_path, endpoints_dict, self.image_value_range, scope=scope) - self.bottlenecks_tensors = PublicImageModelWrapper.get_bottleneck_tensors( - scope) - graph = tf.compat.v1.get_default_graph() - - # Construct gradient ops. - with graph.as_default(): - self.y_input = tf.compat.v1.placeholder(tf.int64, shape=[None]) - - self.pred = tf.expand_dims(self.ends['prediction'][0], 0) - self.loss = tf.reduce_mean( - input_tensor=tf.compat.v1.nn.softmax_cross_entropy_with_logits_v2( - labels=tf.one_hot( - self.y_input, - self.ends['prediction'].get_shape().as_list()[1]), - logits=self.pred)) - self._make_gradient_tensors() - - def id_to_label(self, idx): - return self.labels[idx] - - def label_to_id(self, label): - return self.labels.index(label) - - @staticmethod - def create_input(t_input, image_value_range): - """Create input tensor.""" - def forget_xy(t): - """Forget sizes of dimensions [1, 2] of a 4d tensor.""" - zero = tf.identity(0) - return t[:, zero:, zero:, :] - - t_prep_input = t_input - if len(t_prep_input.shape) == 3: - t_prep_input = tf.expand_dims(t_prep_input, 0) - t_prep_input = forget_xy(t_prep_input) - lo, hi = image_value_range - t_prep_input = lo + t_prep_input * (hi - lo) - return t_input, t_prep_input - - - # From Alex's code. - @staticmethod - def get_bottleneck_tensors(scope): - """Add Inception bottlenecks and their pre-Relu versions to endpoints dict.""" - graph = tf.compat.v1.get_default_graph() - bn_endpoints = {} - for op in graph.get_operations(): - if op.name.startswith(scope + '/') and 'Concat' in op.type: - name = op.name.split('/')[1] - bn_endpoints[name] = op.outputs[0] - return bn_endpoints - - # Load graph and import into graph used by our session - @staticmethod - def import_graph(saved_path, endpoints, image_value_range, scope='import'): - t_input = tf.compat.v1.placeholder(np.float32, [None, None, None, 3]) - graph = tf.Graph() - assert graph.unique_name(scope, False) == scope, ( - 'Scope "%s" already exists. Provide explicit scope names when ' - 'importing multiple instances of the model.') % scope - - graph_def = tf.compat.v1.GraphDef.FromString( - tf.io.gfile.GFile(saved_path, 'rb').read()) - - with tf.compat.v1.name_scope(scope) as sc: - t_input, t_prep_input = PublicImageModelWrapper.create_input( - t_input, image_value_range) - - graph_inputs = {} - graph_inputs[endpoints['input']] = t_prep_input - myendpoints = tf.import_graph_def( - graph_def, graph_inputs, list(endpoints.values()), name=sc) - myendpoints = dict(list(zip(list(endpoints.keys()), myendpoints))) - myendpoints['input'] = t_input - return myendpoints + """Simple wrapper of the public image models with session object.""" + + def __init__( + self, sess, model_fn_path, labels_path, image_shape, endpoints_dict, scope + ): + super(PublicImageModelWrapper, self).__init__(image_shape) + self.labels = tf.io.gfile.GFile(labels_path).read().splitlines() + self.ends = PublicImageModelWrapper.import_graph( + model_fn_path, endpoints_dict, self.image_value_range, scope=scope + ) + self.bottlenecks_tensors = PublicImageModelWrapper.get_bottleneck_tensors(scope) + graph = tf.compat.v1.get_default_graph() + + # Construct gradient ops. + with graph.as_default(): + self.y_input = tf.compat.v1.placeholder(tf.int64, shape=[None]) + + self.pred = tf.expand_dims(self.ends["prediction"][0], 0) + self.loss = tf.reduce_mean( + input_tensor=tf.compat.v1.nn.softmax_cross_entropy_with_logits_v2( + labels=tf.one_hot( + self.y_input, self.ends["prediction"].get_shape().as_list()[1] + ), + logits=self.pred, + ) + ) + self._make_gradient_tensors() + + def id_to_label(self, idx): + return self.labels[idx] + + def label_to_id(self, label): + return self.labels.index(label) + + @staticmethod + def create_input(t_input, image_value_range): + """Create input tensor.""" + + def forget_xy(t): + """Forget sizes of dimensions [1, 2] of a 4d tensor.""" + zero = tf.identity(0) + return t[:, zero:, zero:, :] + + t_prep_input = t_input + if len(t_prep_input.shape) == 3: + t_prep_input = tf.expand_dims(t_prep_input, 0) + t_prep_input = forget_xy(t_prep_input) + lo, hi = image_value_range + t_prep_input = lo + t_prep_input * (hi - lo) + return t_input, t_prep_input + + # From Alex's code. + @staticmethod + def get_bottleneck_tensors(scope): + """Add Inception bottlenecks and their pre-Relu versions to endpoints dict.""" + graph = tf.compat.v1.get_default_graph() + bn_endpoints = {} + for op in graph.get_operations(): + if op.name.startswith(scope + "/") and "Concat" in op.type: + name = op.name.split("/")[1] + bn_endpoints[name] = op.outputs[0] + return bn_endpoints + + # Load graph and import into graph used by our session + @staticmethod + def import_graph(saved_path, endpoints, image_value_range, scope="import"): + t_input = tf.compat.v1.placeholder(np.float32, [None, None, None, 3]) + graph = tf.Graph() + assert graph.unique_name(scope, False) == scope, ( + 'Scope "%s" already exists. Provide explicit scope names when ' + "importing multiple instances of the model." + ) % scope + + graph_def = tf.compat.v1.GraphDef.FromString( + tf.io.gfile.GFile(saved_path, "rb").read() + ) + + with tf.compat.v1.name_scope(scope) as sc: + t_input, t_prep_input = PublicImageModelWrapper.create_input( + t_input, image_value_range + ) + + graph_inputs = {} + graph_inputs[endpoints["input"]] = t_prep_input + myendpoints = tf.import_graph_def( + graph_def, graph_inputs, list(endpoints.values()), name=sc + ) + myendpoints = dict(list(zip(list(endpoints.keys()), myendpoints))) + myendpoints["input"] = t_input + return myendpoints class GoogleNetWrapper_public(PublicImageModelWrapper): + def __init__(self, sess, model_saved_path, labels_path): + image_shape_v1 = [224, 224, 3] + self.image_value_range = (-117, 255 - 117) + endpoints_v1 = dict( + input="input:0", + logit="softmax2_pre_activation:0", + prediction="output2:0", + pre_avgpool="mixed5b:0", + logit_weight="softmax2_w:0", + logit_bias="softmax2_b:0", + ) + self.sess = sess + super(GoogleNetWrapper_public, self).__init__( + sess, + model_saved_path, + labels_path, + image_shape_v1, + endpoints_v1, + scope="v1", + ) + self.model_name = "GoogleNet_public" + + def adjust_prediction(self, pred_t): + # Each pred outputs 16, 1008 matrix. The prediction value is the first row. + # Following tfzoo convention. + return pred_t[::16] - def __init__(self, sess, model_saved_path, labels_path): - image_shape_v1 = [224, 224, 3] - self.image_value_range = (-117, 255 - 117) - endpoints_v1 = dict( - input='input:0', - logit='softmax2_pre_activation:0', - prediction='output2:0', - pre_avgpool='mixed5b:0', - logit_weight='softmax2_w:0', - logit_bias='softmax2_b:0', - ) - self.sess = sess - super(GoogleNetWrapper_public, self).__init__( - sess, - model_saved_path, - labels_path, - image_shape_v1, - endpoints_v1, - scope='v1') - self.model_name = 'GoogleNet_public' - - def adjust_prediction(self, pred_t): - # Each pred outputs 16, 1008 matrix. The prediction value is the first row. - # Following tfzoo convention. - return pred_t[::16] class InceptionV3Wrapper_public(PublicImageModelWrapper): - def __init__(self, sess, model_saved_path, labels_path): - self.image_value_range = (-1, 1) - image_shape_v3 = [299, 299, 3] - endpoints_v3 = dict( - input='Mul:0', - logit='softmax/logits:0', - prediction='softmax:0', - pre_avgpool='mixed_10/join:0', - logit_weight='softmax/weights:0', - logit_bias='softmax/biases:0', - ) - - self.sess = sess - super(InceptionV3Wrapper_public, self).__init__( - sess, - model_saved_path, - labels_path, - image_shape_v3, - endpoints_v3, - scope='v3') - self.model_name = 'InceptionV3_public' + def __init__(self, sess, model_saved_path, labels_path): + self.image_value_range = (-1, 1) + image_shape_v3 = [299, 299, 3] + endpoints_v3 = dict( + input="Mul:0", + logit="softmax/logits:0", + prediction="softmax:0", + pre_avgpool="mixed_10/join:0", + logit_weight="softmax/weights:0", + logit_bias="softmax/biases:0", + ) + + self.sess = sess + super(InceptionV3Wrapper_public, self).__init__( + sess, + model_saved_path, + labels_path, + image_shape_v3, + endpoints_v3, + scope="v3", + ) + self.model_name = "InceptionV3_public" class MobilenetV2Wrapper_public(PublicImageModelWrapper): - - def __init__(self, sess, model_saved_path, labels_path): - self.image_value_range = (-1, 1) - image_shape_v2 = [224, 224, 3] - endpoints_v2 = dict( - input='input:0', - prediction='MobilenetV2/Predictions/Reshape:0', - ) - - self.sess = sess - super(MobilenetV2Wrapper_public, self).__init__( - sess, - model_saved_path, - labels_path, - image_shape_v2, - endpoints_v2, - scope='MobilenetV2') - - # define bottleneck tensors and their gradients - self.bottlenecks_tensors = self.get_bottleneck_tensors_mobilenet( - scope='MobilenetV2') - # Construct gradient ops. - g = tf.compat.v1.get_default_graph() - self._make_gradient_tensors() - self.model_name = 'MobilenetV2_public' - - @staticmethod - def get_bottleneck_tensors_mobilenet(scope): - """Add Inception bottlenecks and their pre-Relu versions to endpoints dict.""" - graph = tf.compat.v1.get_default_graph() - bn_endpoints = {} - for op in graph.get_operations(): - if 'add' in op.name and 'gradients' not in op.name and 'add' == op.name.split( - '/')[-1]: - name = op.name.split('/')[-2] - bn_endpoints[name] = op.outputs[0] - return bn_endpoints + def __init__(self, sess, model_saved_path, labels_path): + self.image_value_range = (-1, 1) + image_shape_v2 = [224, 224, 3] + endpoints_v2 = dict( + input="input:0", + prediction="MobilenetV2/Predictions/Reshape:0", + ) + + self.sess = sess + super(MobilenetV2Wrapper_public, self).__init__( + sess, + model_saved_path, + labels_path, + image_shape_v2, + endpoints_v2, + scope="MobilenetV2", + ) + + # define bottleneck tensors and their gradients + self.bottlenecks_tensors = self.get_bottleneck_tensors_mobilenet( + scope="MobilenetV2" + ) + # Construct gradient ops. + g = tf.compat.v1.get_default_graph() + self._make_gradient_tensors() + self.model_name = "MobilenetV2_public" + + @staticmethod + def get_bottleneck_tensors_mobilenet(scope): + """Add Inception bottlenecks and their pre-Relu versions to endpoints dict.""" + graph = tf.compat.v1.get_default_graph() + bn_endpoints = {} + for op in graph.get_operations(): + if ( + "add" in op.name + and "gradients" not in op.name + and "add" == op.name.split("/")[-1] + ): + name = op.name.split("/")[-2] + bn_endpoints[name] = op.outputs[0] + return bn_endpoints class KerasModelWrapper(ModelWrapper): - """ ModelWrapper for keras models + """ModelWrapper for keras models By default, assumes that your model contains one input node, one output head and one loss function. @@ -431,45 +453,44 @@ class KerasModelWrapper(ModelWrapper): model. You want to make sure that the order of labels in this file matches with the logits layers for your model, such that file[i] == model_logits[i] - """ - - def __init__( - self, - sess, - model_path, - labels_path, - ): - self.sess = sess - super(KerasModelWrapper, self).__init__() - self.import_keras_model(model_path) - self.labels = tf.io.gfile.GFile(labels_path).read().splitlines() - - # Construct gradient ops. Defaults to using the model's output layer - self.y_input = tf.compat.v1.placeholder(tf.int64, shape=[None]) - self.loss = self.model.loss_functions[0](self.y_input, - self.model.outputs[0]) - self._make_gradient_tensors() - - def id_to_label(self, idx): - return self.labels[idx] - - def label_to_id(self, label): - return self.labels.index(label) - - def import_keras_model(self, saved_path): - """Loads keras model, fetching bottlenecks, inputs and outputs.""" - self.ends = {} - self.model = tf.keras.models.load_model(saved_path) - self.get_bottleneck_tensors() - self.get_inputs_and_outputs_and_ends() - - def get_bottleneck_tensors(self): - self.bottlenecks_tensors = {} - layers = self.model.layers - for layer in layers: - if 'input' not in layer.name: - self.bottlenecks_tensors[layer.name] = layer.output - - def get_inputs_and_outputs_and_ends(self): - self.ends['input'] = self.model.inputs[0] - self.ends['prediction'] = self.model.outputs[0] + """ + + def __init__( + self, + sess, + model_path, + labels_path, + ): + self.sess = sess + super(KerasModelWrapper, self).__init__() + self.import_keras_model(model_path) + self.labels = tf.io.gfile.GFile(labels_path).read().splitlines() + + # Construct gradient ops. Defaults to using the model's output layer + self.y_input = tf.compat.v1.placeholder(tf.int64, shape=[None]) + self.loss = self.model.loss_functions[0](self.y_input, self.model.outputs[0]) + self._make_gradient_tensors() + + def id_to_label(self, idx): + return self.labels[idx] + + def label_to_id(self, label): + return self.labels.index(label) + + def import_keras_model(self, saved_path): + """Loads keras model, fetching bottlenecks, inputs and outputs.""" + self.ends = {} + self.model = tf.keras.models.load_model(saved_path) + self.get_bottleneck_tensors() + self.get_inputs_and_outputs_and_ends() + + def get_bottleneck_tensors(self): + self.bottlenecks_tensors = {} + layers = self.model.layers + for layer in layers: + if "input" not in layer.name: + self.bottlenecks_tensors[layer.name] = layer.output + + def get_inputs_and_outputs_and_ends(self): + self.ends["input"] = self.model.inputs[0] + self.ends["prediction"] = self.model.outputs[0] diff --git a/tcav/tcav.py b/tcav/tcav.py index 6fb2cf2cde..95bb3a9713 100644 --- a/tcav/tcav.py +++ b/tcav/tcav.py @@ -29,374 +29,440 @@ class TCAV(object): - """TCAV object: runs TCAV for one target and a set of concepts. - The static methods (get_direction_dir_sign, compute_tcav_score, - get_directional_dir) invole getting directional derivatives and calculating - TCAV scores. These are static because they might be useful independently, - for instance, if you are developing a new interpretability method using - CAVs. - See https://arxiv.org/abs/1711.11279 - """ - - @staticmethod - def get_direction_dir_sign(mymodel, act, cav, concept, class_id, example): - """Get the sign of directional derivative. - - Args: - mymodel: a model class instance - act: activations of one bottleneck to get gradient with respect to. - cav: an instance of cav - concept: one concept - class_id: index of the class of interest (target) in logit layer. - example: example corresponding to the given activation - - Returns: - sign of the directional derivative + """TCAV object: runs TCAV for one target and a set of concepts. + The static methods (get_direction_dir_sign, compute_tcav_score, + get_directional_dir) invole getting directional derivatives and calculating + TCAV scores. These are static because they might be useful independently, + for instance, if you are developing a new interpretability method using + CAVs. + See https://arxiv.org/abs/1711.11279 """ - # Grad points in the direction which DECREASES probability of class - grad = np.reshape(mymodel.get_gradient( - act, [class_id], cav.bottleneck, example), -1) - dot_prod = np.dot(grad, cav.get_direction(concept)) - return dot_prod < 0 - - @staticmethod - def compute_tcav_score(mymodel, - target_class, - concept, - cav, - class_acts, - examples, - run_parallel=True, - num_workers=20): - """Compute TCAV score. - - Args: - mymodel: a model class instance - target_class: one target class - concept: one concept - cav: an instance of cav - class_acts: activations of the examples in the target class where - examples[i] corresponds to class_acts[i] - examples: an array of examples of the target class where examples[i] - corresponds to class_acts[i] - run_parallel: run this parallel fashion - num_workers: number of workers if we run in parallel. - - Returns: - TCAV score (i.e., ratio of pictures that returns negative dot product - wrt loss). - """ - count = 0 - class_id = mymodel.label_to_id(target_class) - if run_parallel: - pool = multiprocessing.Pool(num_workers) - directions = pool.map( - lambda i: TCAV.get_direction_dir_sign( - mymodel, np.expand_dims(class_acts[i], 0), - cav, concept, class_id, examples[i]), - range(len(class_acts))) - pool.close() - return sum(directions) / float(len(class_acts)) - else: - for i in range(len(class_acts)): - act = np.expand_dims(class_acts[i], 0) - example = examples[i] - if TCAV.get_direction_dir_sign( - mymodel, act, cav, concept, class_id, example): - count += 1 - return float(count) / float(len(class_acts)) - - @staticmethod - def get_directional_dir( - mymodel, target_class, concept, cav, class_acts, examples): - """Return the list of values of directional derivatives. - - (Only called when the values are needed as a referece) - - Args: - mymodel: a model class instance - target_class: one target class - concept: one concept - cav: an instance of cav - class_acts: activations of the examples in the target class where - examples[i] corresponds to class_acts[i] - examples: an array of examples of the target class where examples[i] - corresponds to class_acts[i] - - Returns: - list of values of directional derivatives. - """ - class_id = mymodel.label_to_id(target_class) - directional_dir_vals = [] - for i in range(len(class_acts)): - act = np.expand_dims(class_acts[i], 0) - example = examples[i] - grad = np.reshape( - mymodel.get_gradient(act, [class_id], cav.bottleneck, example), -1) - directional_dir_vals.append(np.dot(grad, cav.get_direction(concept))) - return directional_dir_vals - - def __init__(self, - sess, - target, - concepts, - bottlenecks, - activation_generator, - alphas, - random_counterpart=None, - cav_dir=None, - num_random_exp=5, - random_concepts=None): - """Initialze tcav class. - - Args: - sess: tensorflow session. - target: one target class - concepts: A list of names of positive concept sets. - bottlenecks: the name of a bottleneck of interest. - activation_generator: an ActivationGeneratorInterface instance to return - activations. - alphas: list of hyper parameters to run - cav_dir: the path to store CAVs - random_counterpart: the random concept to run against the concepts for - statistical testing. If supplied, only this set will be - used as a positive set for calculating random TCAVs - num_random_exp: number of random experiments to compare against. - random_concepts: A list of names of random concepts for the random - experiments to draw from. Optional, if not provided, the - names will be random500_{i} for i in num_random_exp. - Relative TCAV can be performed by passing in the same - value for both concepts and random_concepts. - """ - self.target = target - self.concepts = concepts - self.bottlenecks = bottlenecks - self.activation_generator = activation_generator - self.cav_dir = cav_dir - self.alphas = alphas - self.mymodel = activation_generator.get_model() - self.model_to_run = self.mymodel.model_name - self.sess = sess - self.random_counterpart = random_counterpart - self.relative_tcav = (random_concepts is not None) and (set(concepts) == set(random_concepts)) - - if num_random_exp < 2: - tf.compat.v1.logging.error('the number of random concepts has to be at least 2') - if random_concepts: - num_random_exp = len(random_concepts) - - # make pairs to test. - self._process_what_to_run_expand(num_random_exp=num_random_exp, - random_concepts=random_concepts) - # parameters - self.params = self.get_params() - tf.compat.v1.logging.info('TCAV will %s params' % len(self.params)) - - def run(self, num_workers=10, run_parallel=False, overwrite=False, return_proto=False): - """Run TCAV for all parameters (concept and random), write results to html. - - Args: - num_workers: number of workers to parallelize - run_parallel: run this parallel. - overwrite: if True, overwrite any saved CAV files. - return_proto: if True, returns results as a tcav.Results object; else, - return as a list of dicts. - - Returns: - results: an object (either a Results proto object or a list of - dictionaries) containing metrics for TCAV results. - """ - # for random exp, a machine with cpu = 30, ram = 300G, disk = 10G and - # pool worker 50 seems to work. - tf.compat.v1.logging.info('running %s params' % len(self.params)) - results = [] - now = time.time() - if run_parallel: - pool = multiprocessing.Pool(num_workers) - for i, res in enumerate(pool.imap( - lambda p: self._run_single_set( - p, overwrite=overwrite, run_parallel=run_parallel), - self.params), 1): - tf.compat.v1.logging.info('Finished running param %s of %s' % (i, len(self.params))) - results.append(res) - pool.close() - else: - for i, param in enumerate(self.params): - tf.compat.v1.logging.info('Running param %s of %s' % (i, len(self.params))) - results.append(self._run_single_set(param, overwrite=overwrite, run_parallel=run_parallel)) - tf.compat.v1.logging.info('Done running %s params. Took %s seconds...' % (len( - self.params), time.time() - now)) - if return_proto: - return utils.results_to_proto(results) - else: - return results - - def _run_single_set(self, param, overwrite=False, run_parallel=False): - """Run TCAV with provided for one set of (target, concepts). - - Args: - param: parameters to run - overwrite: if True, overwrite any saved CAV files. - run_parallel: run this parallel. - - Returns: - a dictionary of results (panda frame) - """ - bottleneck = param.bottleneck - concepts = param.concepts - target_class = param.target_class - activation_generator = param.activation_generator - alpha = param.alpha - mymodel = param.model - cav_dir = param.cav_dir - # first check if target class is in model. - - tf.compat.v1.logging.info('running %s %s' % (target_class, concepts)) - - # Get acts - acts = activation_generator.process_and_load_activations( - [bottleneck], concepts + [target_class]) - # Get CAVs - cav_hparams = CAV.default_hparams() - cav_hparams['alpha'] = alpha - cav_instance = get_or_train_cav( + + @staticmethod + def get_direction_dir_sign(mymodel, act, cav, concept, class_id, example): + """Get the sign of directional derivative. + + Args: + mymodel: a model class instance + act: activations of one bottleneck to get gradient with respect to. + cav: an instance of cav + concept: one concept + class_id: index of the class of interest (target) in logit layer. + example: example corresponding to the given activation + + Returns: + sign of the directional derivative + """ + # Grad points in the direction which DECREASES probability of class + grad = np.reshape( + mymodel.get_gradient(act, [class_id], cav.bottleneck, example), -1 + ) + dot_prod = np.dot(grad, cav.get_direction(concept)) + + return dot_prod < 0 + + @staticmethod + def compute_tcav_score( + mymodel, + target_class, + concept, + cav, + class_acts, + examples, + run_parallel=True, + num_workers=20, + ): + """Compute TCAV score. + + Args: + mymodel: a model class instance + target_class: one target class + concept: one concept + cav: an instance of cav + class_acts: activations of the examples in the target class where + examples[i] corresponds to class_acts[i] + examples: an array of examples of the target class where examples[i] + corresponds to class_acts[i] + run_parallel: run this parallel fashion + num_workers: number of workers if we run in parallel. + + Returns: + TCAV score (i.e., ratio of pictures that returns negative dot product + wrt loss). + """ + count = 0 + class_id = mymodel.label_to_id(target_class) + if run_parallel: + pool = multiprocessing.Pool(num_workers) + directions = pool.map( + lambda i: TCAV.get_direction_dir_sign( + mymodel, + np.expand_dims(class_acts[i], 0), + cav, + concept, + class_id, + examples[i], + ), + range(len(class_acts)), + ) + pool.close() + return sum(directions) / float(len(class_acts)) + else: + for i in range(len(class_acts)): + act = np.expand_dims(class_acts[i], 0) + example = examples[i] + if TCAV.get_direction_dir_sign( + mymodel, act, cav, concept, class_id, example + ): + count += 1 + return float(count) / float(len(class_acts)) + + @staticmethod + def get_directional_dir(mymodel, target_class, concept, cav, class_acts, examples): + """Return the list of values of directional derivatives. + + (Only called when the values are needed as a referece) + + Args: + mymodel: a model class instance + target_class: one target class + concept: one concept + cav: an instance of cav + class_acts: activations of the examples in the target class where + examples[i] corresponds to class_acts[i] + examples: an array of examples of the target class where examples[i] + corresponds to class_acts[i] + + Returns: + list of values of directional derivatives. + """ + class_id = mymodel.label_to_id(target_class) + directional_dir_vals = [] + for i in range(len(class_acts)): + act = np.expand_dims(class_acts[i], 0) + example = examples[i] + grad = np.reshape( + mymodel.get_gradient(act, [class_id], cav.bottleneck, example), -1 + ) + directional_dir_vals.append(np.dot(grad, cav.get_direction(concept))) + return directional_dir_vals + + def __init__( + self, + sess, + target, concepts, - bottleneck, - acts, - cav_dir=cav_dir, - cav_hparams=cav_hparams, - overwrite=overwrite) - - # clean up - for c in concepts: - del acts[c] - - # Hypo testing - a_cav_key = CAV.cav_key(concepts, bottleneck, cav_hparams['model_type'], - cav_hparams['alpha']) - target_class_for_compute_tcav_score = target_class - - cav_concept = concepts[0] - - i_up = self.compute_tcav_score( - mymodel, target_class_for_compute_tcav_score, cav_concept, - cav_instance, acts[target_class][cav_instance.bottleneck], - activation_generator.get_examples_for_concept(target_class), - run_parallel=run_parallel) - val_directional_dirs = self.get_directional_dir( - mymodel, target_class_for_compute_tcav_score, cav_concept, - cav_instance, acts[target_class][cav_instance.bottleneck], - activation_generator.get_examples_for_concept(target_class)) - result = { - 'cav_key': - a_cav_key, - 'cav_concept': + bottlenecks, + activation_generator, + alphas, + random_counterpart=None, + cav_dir=None, + num_random_exp=5, + random_concepts=None, + ): + """Initialze tcav class. + + Args: + sess: tensorflow session. + target: one target class + concepts: A list of names of positive concept sets. + bottlenecks: the name of a bottleneck of interest. + activation_generator: an ActivationGeneratorInterface instance to return + activations. + alphas: list of hyper parameters to run + cav_dir: the path to store CAVs + random_counterpart: the random concept to run against the concepts for + statistical testing. If supplied, only this set will be + used as a positive set for calculating random TCAVs + num_random_exp: number of random experiments to compare against. + random_concepts: A list of names of random concepts for the random + experiments to draw from. Optional, if not provided, the + names will be random500_{i} for i in num_random_exp. + Relative TCAV can be performed by passing in the same + value for both concepts and random_concepts. + """ + self.target = target + self.concepts = concepts + self.bottlenecks = bottlenecks + self.activation_generator = activation_generator + self.cav_dir = cav_dir + self.alphas = alphas + self.mymodel = activation_generator.get_model() + self.model_to_run = self.mymodel.model_name + self.sess = sess + self.random_counterpart = random_counterpart + self.relative_tcav = (random_concepts is not None) and ( + set(concepts) == set(random_concepts) + ) + + if num_random_exp < 2: + tf.compat.v1.logging.error( + "the number of random concepts has to be at least 2" + ) + if random_concepts: + num_random_exp = len(random_concepts) + + # make pairs to test. + self._process_what_to_run_expand( + num_random_exp=num_random_exp, random_concepts=random_concepts + ) + # parameters + self.params = self.get_params() + tf.compat.v1.logging.info("TCAV will %s params" % len(self.params)) + + def run( + self, num_workers=10, run_parallel=False, overwrite=False, return_proto=False + ): + """Run TCAV for all parameters (concept and random), write results to html. + + Args: + num_workers: number of workers to parallelize + run_parallel: run this parallel. + overwrite: if True, overwrite any saved CAV files. + return_proto: if True, returns results as a tcav.Results object; else, + return as a list of dicts. + + Returns: + results: an object (either a Results proto object or a list of + dictionaries) containing metrics for TCAV results. + """ + # for random exp, a machine with cpu = 30, ram = 300G, disk = 10G and + # pool worker 50 seems to work. + tf.compat.v1.logging.info("running %s params" % len(self.params)) + results = [] + now = time.time() + if run_parallel: + pool = multiprocessing.Pool(num_workers) + for i, res in enumerate( + pool.imap( + lambda p: self._run_single_set( + p, overwrite=overwrite, run_parallel=run_parallel + ), + self.params, + ), + 1, + ): + tf.compat.v1.logging.info( + "Finished running param %s of %s" % (i, len(self.params)) + ) + results.append(res) + pool.close() + else: + for i, param in enumerate(self.params): + tf.compat.v1.logging.info( + "Running param %s of %s" % (i, len(self.params)) + ) + # tf.compat.v1.logging.info("Hello: We are at %s" % (vars(param))) #print parameters + results.append( + self._run_single_set( + param, overwrite=overwrite, run_parallel=run_parallel + ) + ) + tf.compat.v1.logging.info( + "Done running %s params. Took %s seconds..." + % (len(self.params), time.time() - now) + ) + if return_proto: + return utils.results_to_proto(results) + else: + return results + + def _run_single_set(self, param, overwrite=False, run_parallel=False): + """Run TCAV with provided for one set of (target, concepts). + + Args: + param: parameters to run + overwrite: if True, overwrite any saved CAV files. + run_parallel: run this parallel. + + Returns: + a dictionary of results (panda frame) + """ + bottleneck = param.bottleneck + concepts = param.concepts + target_class = param.target_class + activation_generator = param.activation_generator + alpha = param.alpha + mymodel = param.model + cav_dir = param.cav_dir + # first check if target class is in model. + + tf.compat.v1.logging.info("running %s %s" % (target_class, concepts)) + + # Get acts + acts = activation_generator.process_and_load_activations( + [bottleneck], concepts + [target_class] + ) + # Get CAVs + cav_hparams = CAV.default_hparams() + cav_hparams["alpha"] = alpha + cav_instance = get_or_train_cav( + concepts, + bottleneck, + acts, + cav_dir=cav_dir, + cav_hparams=cav_hparams, + overwrite=overwrite, + ) + + # clean up + for c in concepts: + del acts[c] + + # Hypo testing + a_cav_key = CAV.cav_key( + concepts, bottleneck, cav_hparams["model_type"], cav_hparams["alpha"] + ) + target_class_for_compute_tcav_score = target_class + + cav_concept = concepts[0] + + i_up = self.compute_tcav_score( + mymodel, + target_class_for_compute_tcav_score, cav_concept, - 'negative_concept': - concepts[1], - 'target_class': - target_class, - 'cav_accuracies': - cav_instance.accuracies, - 'i_up': - i_up, - 'val_directional_dirs_abs_mean': - np.mean(np.abs(val_directional_dirs)), - 'val_directional_dirs_mean': - np.mean(val_directional_dirs), - 'val_directional_dirs_std': - np.std(val_directional_dirs), - 'val_directional_dirs': - val_directional_dirs, - 'note': - 'alpha_%s ' % (alpha), - 'alpha': - alpha, - 'bottleneck': - bottleneck - } - del acts - return result - - def _process_what_to_run_expand(self, num_random_exp=100, random_concepts=None): - """Get tuples of parameters to run TCAV with. - - TCAV builds random concept to conduct statistical significance testing - againts the concept. To do this, we build many concept vectors, and many - random vectors. This function prepares runs by expanding parameters. - - Args: - num_random_exp: number of random experiments to run to compare. - random_concepts: A list of names of random concepts for the random experiments - to draw from. Optional, if not provided, the names will be - random500_{i} for i in num_random_exp. - """ - - target_concept_pairs = [(self.target, self.concepts)] - - # take away 1 random experiment if the random counterpart already in random concepts - # take away 1 random experiment if computing Relative TCAV - all_concepts_concepts, pairs_to_run_concepts = ( - utils.process_what_to_run_expand( + cav_instance, + acts[target_class][cav_instance.bottleneck], + activation_generator.get_examples_for_concept(target_class), + run_parallel=run_parallel, + ) + val_directional_dirs = self.get_directional_dir( + mymodel, + target_class_for_compute_tcav_score, + cav_concept, + cav_instance, + acts[target_class][cav_instance.bottleneck], + activation_generator.get_examples_for_concept(target_class), + ) + result = { + "cav_key": a_cav_key, + "cav_concept": cav_concept, + "negative_concept": concepts[1], + "target_class": target_class, + "cav_accuracies": cav_instance.accuracies, + "i_up": i_up, + "val_directional_dirs_abs_mean": np.mean(np.abs(val_directional_dirs)), + "val_directional_dirs_mean": np.mean(val_directional_dirs), + "val_directional_dirs_std": np.std(val_directional_dirs), + "val_directional_dirs": val_directional_dirs, + "note": "alpha_%s " % (alpha), + "alpha": alpha, + "bottleneck": bottleneck, + } + del acts + return result + + def _process_what_to_run_expand(self, num_random_exp=100, random_concepts=None): + """Get tuples of parameters to run TCAV with. + + TCAV builds random concept to conduct statistical significance testing + againts the concept. To do this, we build many concept vectors, and many + random vectors. This function prepares runs by expanding parameters. + + Args: + num_random_exp: number of random experiments to run to compare. + random_concepts: A list of names of random concepts for the random experiments + to draw from. Optional, if not provided, the names will be + random500_{i} for i in num_random_exp. + """ + + target_concept_pairs = [(self.target, self.concepts)] + + # take away 1 random experiment if the random counterpart already in random concepts + # take away 1 random experiment if computing Relative TCAV + all_concepts_concepts, pairs_to_run_concepts = utils.process_what_to_run_expand( utils.process_what_to_run_concepts(target_concept_pairs), self.random_counterpart, - num_random_exp=num_random_exp - - (1 if random_concepts and self.random_counterpart in random_concepts - else 0) - (1 if self.relative_tcav else 0), - random_concepts=random_concepts)) - - pairs_to_run_randoms = [] - all_concepts_randoms = [] - - # ith random concept - def get_random_concept(i): - return (random_concepts[i] if random_concepts - else 'random500_{}'.format(i)) - - if self.random_counterpart is None: - # TODO random500_1 vs random500_0 is the same as 1 - (random500_0 vs random500_1) - for i in range(num_random_exp): - all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = ( - utils.process_what_to_run_expand( - utils.process_what_to_run_randoms(target_concept_pairs, - get_random_concept(i)), - num_random_exp=num_random_exp - 1, - random_concepts=random_concepts)) - - pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) - all_concepts_randoms.extend(all_concepts_randoms_tmp) - - else: - # run only random_counterpart as the positve set for random experiments - all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = ( - utils.process_what_to_run_expand( - utils.process_what_to_run_randoms(target_concept_pairs, - self.random_counterpart), - self.random_counterpart, - num_random_exp=num_random_exp - (1 if random_concepts and - self.random_counterpart in random_concepts else 0), - random_concepts=random_concepts)) - - pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) - all_concepts_randoms.extend(all_concepts_randoms_tmp) - - self.all_concepts = list(set(all_concepts_concepts + all_concepts_randoms)) - self.pairs_to_test = pairs_to_run_concepts if self.relative_tcav else pairs_to_run_concepts + pairs_to_run_randoms - - def get_params(self): - """Enumerate parameters for the run function. - - Returns: - parameters - """ - params = [] - for bottleneck in self.bottlenecks: - for target_in_test, concepts_in_test in self.pairs_to_test: - for alpha in self.alphas: - tf.compat.v1.logging.info('%s %s %s %s', bottleneck, concepts_in_test, - target_in_test, alpha) - params.append( - run_params.RunParams(bottleneck, concepts_in_test, target_in_test, - self.activation_generator, self.cav_dir, - alpha, self.mymodel)) - return params + num_random_exp=num_random_exp + - ( + 1 + if random_concepts and self.random_counterpart in random_concepts + else 0 + ) + - (1 if self.relative_tcav else 0), + random_concepts=random_concepts, + ) + + pairs_to_run_randoms = [] + all_concepts_randoms = [] + + # ith random concept + def get_random_concept(i): + return random_concepts[i] if random_concepts else "random500_{}".format(i) + + if self.random_counterpart is None: + # TODO random500_1 vs random500_0 is the same as 1 - (random500_0 vs random500_1) + for i in range(num_random_exp): + ( + all_concepts_randoms_tmp, + pairs_to_run_randoms_tmp, + ) = utils.process_what_to_run_expand( + utils.process_what_to_run_randoms( + target_concept_pairs, get_random_concept(i) + ), + num_random_exp=num_random_exp - 1, + random_concepts=random_concepts, + ) + + pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) + all_concepts_randoms.extend(all_concepts_randoms_tmp) + + else: + # run only random_counterpart as the positve set for random experiments + ( + all_concepts_randoms_tmp, + pairs_to_run_randoms_tmp, + ) = utils.process_what_to_run_expand( + utils.process_what_to_run_randoms( + target_concept_pairs, self.random_counterpart + ), + self.random_counterpart, + num_random_exp=num_random_exp + - ( + 1 + if random_concepts and self.random_counterpart in random_concepts + else 0 + ), + random_concepts=random_concepts, + ) + + pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp) + all_concepts_randoms.extend(all_concepts_randoms_tmp) + + self.all_concepts = list(set(all_concepts_concepts + all_concepts_randoms)) + self.pairs_to_test = ( + pairs_to_run_concepts + if self.relative_tcav + else pairs_to_run_concepts + pairs_to_run_randoms + ) + + def get_params(self): + """Enumerate parameters for the run function. + + Returns: + parameters + """ + params = [] + for bottleneck in self.bottlenecks: + for target_in_test, concepts_in_test in self.pairs_to_test: + for alpha in self.alphas: + tf.compat.v1.logging.info( + "%s %s %s %s", + bottleneck, + concepts_in_test, + target_in_test, + alpha, + ) + params.append( + run_params.RunParams( + bottleneck, + concepts_in_test, + target_in_test, + self.activation_generator, + self.cav_dir, + alpha, + self.mymodel, + ) + ) + + return params diff --git a/unpackplkFiles.ipynb b/unpackplkFiles.ipynb new file mode 100644 index 0000000000..db99ac59dd --- /dev/null +++ b/unpackplkFiles.ipynb @@ -0,0 +1,88 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "\n", + "user = 'beenkim'\n", + "project_name = 'tcav_class_test'\n", + "working_dir = \"/tmp/\" + user + '/' + project_name\n", + "activation_dir = working_dir+ '/activations/'\n", + "cav_dir = working_dir + '/cavs/'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "with open(cav_dir + 'dotted-random500_0-mixed4d-linear-0.1.pkl', 'rb') as f:\n", + " data = pickle.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'concepts': ['dotted', 'random500_0'],\n", + " 'bottleneck': 'mixed4d',\n", + " 'hparams': {'model_type': 'linear',\n", + " 'alpha': 0.1,\n", + " 'max_iter': 1000,\n", + " 'tol': 0.001},\n", + " 'accuracies': {'dotted': 1.0, 'random500_0': 1.0, 'overall': 1.0},\n", + " 'cavs': [array([-3.83513699, -0. , -3.15086947, ..., -0. ,\n", + " -0. , 0.20657002]),\n", + " array([ 3.83513699, 0. , 3.15086947, ..., 0. ,\n", + " 0. , -0.20657002])],\n", + " 'saved_path': '/tmp/beenkim/tcav_class_test/cavs/dotted-random500_0-mixed4d-linear-0.1.pkl'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.2 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.2" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}