diff --git a/.gitignore b/.gitignore index 5f7bafda..e2634422 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/source/modules/_autosummary # PyBuilder target/ diff --git a/.readthedocs.yml b/.readthedocs.yml index 57a75f48..529b5306 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,9 +1,14 @@ version: 2 build: - os: "ubuntu-22.04" - tools: - python: "3.8" + os: "ubuntu-22.04" + tools: + python: "3.8" + jobs: + pre_build: + - cat brainscore_language/utils/document_plugins.py + - python brainscore_language/utils/document_plugins.py + python: install: diff --git a/brainscore_language/utils/document_plugins.py b/brainscore_language/utils/document_plugins.py new file mode 100644 index 00000000..1cf0bbef --- /dev/null +++ b/brainscore_language/utils/document_plugins.py @@ -0,0 +1,24 @@ +""" Make plugin details available to readthedocs """ + +from pathlib import Path +from brainscore_core.plugin_management.plugin_utils import get_all_plugin_info +from brainscore_core.plugin_management.document_plugins import create_bibfile, update_readthedocs + + +BIBS_DIR = Path(Path(__file__).parents[2], 'docs', 'source', 'bibtex') +PLUGINS_DOC = Path(Path(__file__).parents[2], 'docs', 'source', 'modules') +GITHUB_DIR = 'https://github.com/brain-score/language/tree/main/brainscore_language/' + + +def update_docs(): + all_plugin_info = get_all_plugin_info(Path(__file__).parents[1]) + print('running document_plugins.py') + print(f"all_plugin_info: {all_plugin_info}") + for plugin_type in all_plugin_info: + create_bibfile(all_plugin_info[plugin_type], BIBS_DIR, plugin_type) # plugin type .bib file + create_bibfile(all_plugin_info, BIBS_DIR) # one .bib file to rule them all + update_readthedocs(all_plugin_info, PLUGINS_DOC, GITHUB_DIR) + + +if __name__ == '__main__': + update_docs() diff --git a/brainscore_language/utils/xarray.py b/brainscore_language/utils/xarray_utils.py similarity index 100% rename from brainscore_language/utils/xarray.py rename to brainscore_language/utils/xarray_utils.py diff --git a/docs/Makefile b/docs/Makefile index 92dd33a1..94265032 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,20 +1,14 @@ # Minimal makefile for Sphinx documentation -# -# You can set these variables from the command line, and also -# from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = _build -# Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.txt b/docs/requirements.txt index 23532fd2..d384c309 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,4 +2,6 @@ Sphinx>=4 sphinx_rtd_theme -recommonmark \ No newline at end of file +recommonmark +sphinxcontrib-bibtex +rstcloth \ No newline at end of file diff --git a/docs/source/bibtex/data.bib b/docs/source/bibtex/data.bib new file mode 100644 index 00000000..589d0bf9 --- /dev/null +++ b/docs/source/bibtex/data.bib @@ -0,0 +1,40 @@ +@article{blank2014functional, + title={A functional dissociation between language and multiple-demand systems revealed in patterns of BOLD signal fluctuations}, + author={Blank, Idan and Kanwisher, Nancy and Fedorenko, Evelina}, + journal={Journal of neurophysiology}, + volume={112}, + number={5}, + pages={1105--1118}, + year={2014}, + publisher={American Physiological Society Bethesda, MD} +} +@article{fedorenko2016neural, + title={Neural correlate of the construction of sentence meaning}, + author={Fedorenko, Evelina and Scott, Terri L and Brunner, Peter and Coon, William G and Pritchett, Brianna and + Schalk, Gerwin and Kanwisher, Nancy}, + journal={Proceedings of the National Academy of Sciences}, + volume={113}, + number={41}, + pages={E6256--E6262}, + year={2016}, + publisher={National Acad Sciences} +} +@proceedings{futrell2018natural, + title={The Natural Stories Corpus}, + author={Futrell, Richard and Gibson, Edward and Tily, Harry J. and Blank, Idan and Vishnevetsky, Anastasia and + Piantadosi, Steven T. and Fedorenko, Evelina}, + conference={International Conference on Language Resources and Evaluation (LREC)}, + url={http://www.lrec-conf.org/proceedings/lrec2018/pdf/337.pdf}, + year={2018} +} +@article{pereira2018toward, + title={Toward a universal decoder of linguistic meaning from brain activation}, + author={Pereira, Francisco and Lou, Bin and Pritchett, Brianna and Ritter, Samuel and Gershman, Samuel J + and Kanwisher, Nancy and Botvinick, Matthew and Fedorenko, Evelina}, + journal={Nature communications}, + volume={9}, + number={1}, + pages={1--13}, + year={2018}, + publisher={Nature Publishing Group} +} diff --git a/docs/source/bibtex/refs.bib b/docs/source/bibtex/refs.bib new file mode 100644 index 00000000..589d0bf9 --- /dev/null +++ b/docs/source/bibtex/refs.bib @@ -0,0 +1,40 @@ +@article{blank2014functional, + title={A functional dissociation between language and multiple-demand systems revealed in patterns of BOLD signal fluctuations}, + author={Blank, Idan and Kanwisher, Nancy and Fedorenko, Evelina}, + journal={Journal of neurophysiology}, + volume={112}, + number={5}, + pages={1105--1118}, + year={2014}, + publisher={American Physiological Society Bethesda, MD} +} +@article{fedorenko2016neural, + title={Neural correlate of the construction of sentence meaning}, + author={Fedorenko, Evelina and Scott, Terri L and Brunner, Peter and Coon, William G and Pritchett, Brianna and + Schalk, Gerwin and Kanwisher, Nancy}, + journal={Proceedings of the National Academy of Sciences}, + volume={113}, + number={41}, + pages={E6256--E6262}, + year={2016}, + publisher={National Acad Sciences} +} +@proceedings{futrell2018natural, + title={The Natural Stories Corpus}, + author={Futrell, Richard and Gibson, Edward and Tily, Harry J. and Blank, Idan and Vishnevetsky, Anastasia and + Piantadosi, Steven T. and Fedorenko, Evelina}, + conference={International Conference on Language Resources and Evaluation (LREC)}, + url={http://www.lrec-conf.org/proceedings/lrec2018/pdf/337.pdf}, + year={2018} +} +@article{pereira2018toward, + title={Toward a universal decoder of linguistic meaning from brain activation}, + author={Pereira, Francisco and Lou, Bin and Pritchett, Brianna and Ritter, Samuel and Gershman, Samuel J + and Kanwisher, Nancy and Botvinick, Matthew and Fedorenko, Evelina}, + journal={Nature communications}, + volume={9}, + number={1}, + pages={1--13}, + year={2018}, + publisher={Nature Publishing Group} +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 61b44176..9ced09c1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,6 +35,8 @@ 'recommonmark', 'sphinx.ext.viewcode', 'sphinx_rtd_theme', + 'sphinxcontrib.bibtex', + 'sphinx.ext.autosectionlabel' ] autosummary_generate = True # Turn on sphinx.ext.autosummary @@ -53,6 +55,9 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] +# allows BibTeX citations to be inserted into docs +bibtex_bibfiles = ['bibtex/refs.bib'] + # -- Options for HTML output ------------------------------------------------- @@ -61,11 +66,6 @@ # html_theme = "sphinx_rtd_theme" -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - def skip(app, what, name, obj, would_skip, options): if name in ("__init__", "__call__"): diff --git a/docs/source/index.rst b/docs/source/index.rst index c137d24f..5675b826 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,4 +25,5 @@ The `score` function is the primary entry point to score a model on a benchmark. modules/benchmark_tutorial examples modules/api_reference + modules/plugins Glossary diff --git a/docs/source/modules/benchmark_tutorial.rst b/docs/source/modules/benchmark_tutorial.rst index b5f09446..01e016f1 100644 --- a/docs/source/modules/benchmark_tutorial.rst +++ b/docs/source/modules/benchmark_tutorial.rst @@ -208,7 +208,7 @@ For example: def __call__(self, candidate: ArtificialSubject) -> Score: candidate.start_behavioral_task(ArtificialSubject.Task.reading_times) # or any other task - # or e.g. candidate.start_start_recording(recording_target=ArtificialSubject.RecordingTarget.language_system, + # or e.g. candidate.start_neural_recording(recording_target=ArtificialSubject.RecordingTarget.language_system, # recording_type=ArtificialSubject.RecordingType.fMRI) predictions = candidate.digest_text(stimuli)['behavior'] raw_score = self.metric(predictions, self.data) diff --git a/docs/source/modules/plugins.rst b/docs/source/modules/plugins.rst new file mode 100644 index 00000000..b6a6bb8a --- /dev/null +++ b/docs/source/modules/plugins.rst @@ -0,0 +1,592 @@ +.. _plugins: + +======= +Plugins +======= + + +Benchmarks +~~~~~~~~~~ +The following Benchmarks are available. Plugin identifiers are grouped +by directory, shown in bold. + +Futrell2018 ++++++++++++ +Location: `benchmarks/futrell2018 +`_ + + +.. _Futrell2018-pearsonr: +Futrell2018-pearsonr +;;;;;;;;;;;;;;;;;;;; + + Data: :ref:`Futrell2018 ` + + Metric: :ref:`pearsonr ` + + +Pereira2018 ++++++++++++ +Location: `benchmarks/pereira2018 +`_ + + +.. _Pereira2018.243sentences-linear: +Pereira2018.243sentences-linear +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Data: :ref:`Pereira2018.language ` + + Metric: :ref:`linear_pearsonr ` + + + +.. _Pereira2018.384sentences-linear: +Pereira2018.384sentences-linear +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Data: :ref:`Pereira2018.language ` + + Metric: :ref:`linear_pearsonr ` + + +Wikitext_next_word +++++++++++++++++++ +Location: `benchmarks/wikitext_next_word +`_ + + +.. _Wikitext-accuracy: +Wikitext-accuracy +;;;;;;;;;;;;;;;;; + + Data: :ref:`wikitext-2/test ` + + Metric: :ref:`accuracy ` + + +Blank2014 ++++++++++ +Location: `benchmarks/blank2014 +`_ + + +.. _Blank2014-linear: +Blank2014-linear +;;;;;;;;;;;;;;;; + + Data: :ref:`Blank2014.fROI ` + + Metric: :ref:`linear_pearsonr ` + + +Syntaxgym ++++++++++ +Location: `benchmarks/syntaxgym +`_ + + +.. _syntaxgym-center_embed: +syntaxgym-center_embed +;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-center_embed_mod: +syntaxgym-center_embed_mod +;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-cleft: +syntaxgym-cleft +;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-cleft_modifier: +syntaxgym-cleft_modifier +;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_hierarchy: +syntaxgym-fgd_hierarchy +;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_object: +syntaxgym-fgd_object +;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_pp: +syntaxgym-fgd_pp +;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-fgd_subject: +syntaxgym-fgd_subject +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-mvrr: +syntaxgym-mvrr +;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-mvrr_mod: +syntaxgym-mvrr_mod +;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_orc_any: +syntaxgym-npi_orc_any +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_orc_ever: +syntaxgym-npi_orc_ever +;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_src_any: +syntaxgym-npi_src_any +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npi_src_ever: +syntaxgym-npi_src_ever +;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_ambig: +syntaxgym-npz_ambig +;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_ambig_mod: +syntaxgym-npz_ambig_mod +;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_obj: +syntaxgym-npz_obj +;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-npz_obj_mod: +syntaxgym-npz_obj_mod +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-number_orc: +syntaxgym-number_orc +;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-number_prep: +syntaxgym-number_prep +;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-number_src: +syntaxgym-number_src +;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_orc_fem: +syntaxgym-reflexive_orc_fem +;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_orc_masc: +syntaxgym-reflexive_orc_masc +;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_prep_fem: +syntaxgym-reflexive_prep_fem +;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_prep_masc: +syntaxgym-reflexive_prep_masc +;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_src_fem: +syntaxgym-reflexive_src_fem +;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-reflexive_src_masc: +syntaxgym-reflexive_src_masc +;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination: +syntaxgym-subordination +;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination_orc-orc: +syntaxgym-subordination_orc-orc +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination_pp-pp: +syntaxgym-subordination_pp-pp +;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +.. _syntaxgym-subordination_src-src: +syntaxgym-subordination_src-src +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + Metric: :ref:`accuracy ` + + + +Data +~~~~ +The following Data are available. Plugin identifiers are grouped by +directory, shown in bold. + +Blank2014 ++++++++++ +Location: `data/blank2014 +`_ + + +.. _Blank2014.fROI: +Blank2014.fROI +;;;;;;;;;;;;;; + + Citation: :cite:label:`blank2014functional` + + +Fedorenko2016 ++++++++++++++ +Location: `data/fedorenko2016 +`_ + + +.. _Fedorenko2016.language: +Fedorenko2016.language +;;;;;;;;;;;;;;;;;;;;;; + + Citation: :cite:label:`fedorenko2016neural` + + +Futrell2018 ++++++++++++ +Location: `data/futrell2018 +`_ + + +.. _Futrell2018: +Futrell2018 +;;;;;;;;;;; + + Citation: :cite:label:`futrell2018natural` + + +Pereira2018 ++++++++++++ +Location: `data/pereira2018 +`_ + + +.. _Pereira2018.language: +Pereira2018.language +;;;;;;;;;;;;;;;;;;;; + + Citation: :cite:label:`pereira2018toward` + + + +.. _Pereira2018.auditory: +Pereira2018.auditory +;;;;;;;;;;;;;;;;;;;; + + Citation: :cite:label:`pereira2018toward` + + +Wikitext +++++++++ +Location: `data/wikitext +`_ + + +.. _wikitext-2/test: +wikitext-2/test +;;;;;;;;;;;;;;; + + + +Metrics +~~~~~~~ +The following Metrics are available. Plugin identifiers are grouped by +directory, shown in bold. + +Accuracy +++++++++ +Location: `metrics/accuracy +`_ + + +.. _accuracy: +accuracy +;;;;;;;; + + +Linear_predictivity ++++++++++++++++++++ +Location: `metrics/linear_predictivity +`_ + + +.. _linear_pearsonr: +linear_pearsonr +;;;;;;;;;;;;;;; + + +Pearson_correlation ++++++++++++++++++++ +Location: `metrics/pearson_correlation +`_ + + +.. _pearsonr: +pearsonr +;;;;;;;; + + + +Models +~~~~~~ +The following Models are available. Plugin identifiers are grouped by +directory, shown in bold. + +Glove ++++++ +Location: `models/glove +`_ + + +.. _glove-840b: +glove-840b +;;;;;;;;;; + + +Gpt ++++ +Location: `models/gpt +`_ + + +.. _distilgpt2: +distilgpt2 +;;;;;;;;;; + + + +.. _gpt2-xl: +gpt2-xl +;;;;;;; + + + +.. _gpt-neo-2.7B: +gpt-neo-2.7B +;;;;;;;;;;;; + + + +.. _gpt-neo-1.3B: +gpt-neo-1.3B +;;;;;;;;;;;; + + +Earley_parser ++++++++++++++ +Location: `models/earley_parser +`_ + + +.. _earley-parser-minivocab: +earley-parser-minivocab +;;;;;;;;;;;;;;;;;;;;;;; + + +Lm1b +++++ +Location: `models/lm1b +`_ + + +.. _lm1b: +lm1b +;;;; + + +Random_embedding +++++++++++++++++ +Location: `models/random_embedding +`_ + + +.. _randomembedding-1600: +randomembedding-1600 +;;;;;;;;;;;;;;;;;;;; + + + +.. _randomembedding-100: +randomembedding-100 +;;;;;;;;;;;;;;;;;;; + + +Rnng +++++ +Location: `models/rnng +`_ + + +.. _rnn-slm-ptb: +rnn-slm-ptb +;;;;;;;;;;; + + + +.. _rnn-tdg-ptb: +rnn-tdg-ptb +;;;;;;;;;;; + + + +.. _rnn-slm-ptboanc: +rnn-slm-ptboanc +;;;;;;;;;;;;;;; + + + +.. _rnn-tdg-ptboanc: +rnn-tdg-ptboanc +;;;;;;;;;;;;;;; + + + +.. _rnn-slm-ptboanc-1024: +rnn-slm-ptboanc-1024 +;;;;;;;;;;;;;;;;;;;; + + + +.. _rnn-tdg-ptboanc-1024: +rnn-tdg-ptboanc-1024 +;;;;;;;;;;;;;;;;;;;; + + +Rnnlm ++++++ +Location: `models/rnnlm +`_ + + +.. _rnn-lm-ptb: +rnn-lm-ptb +;;;;;;;;;; + + +Bibliography +------------ +.. bibliography:: + :all: