From bfe4ab4179241a22a9ee39ef638624fa5e5f5837 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Sun, 4 Aug 2019 15:24:24 -0500 Subject: [PATCH 1/4] Clean-room DNC implementation --- LICENSE | 221 ++-------------- Manifest.toml | 373 ++++++++++++++++++++++++++++ Project.toml | 12 + src/DifferentiableNeuralComputer.jl | 12 + test/runtests.jl | 5 + 5 files changed, 421 insertions(+), 202 deletions(-) create mode 100644 Manifest.toml create mode 100644 src/DifferentiableNeuralComputer.jl create mode 100644 test/runtests.jl diff --git a/LICENSE b/LICENSE index d645695..46490f7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,202 +1,19 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Copyright (c) 2019 Julian P. Samaroo and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..2f11e11 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,373 @@ +# This file is machine-generated - editing it directly is not advised + +[[AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "380e36c66edfa099cd90116b24c1ce8cafccac40" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "0.4.1" + +[[AbstractTrees]] +deps = ["Markdown", "Test"] +git-tree-sha1 = "6621d9645702c1c4e6970cc6a3eae440c768000b" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.2.1" + +[[Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "82dab828020b872fa9efd3abec1152b075bc7cbf" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "1.0.0" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[BinDeps]] +deps = ["Compat", "Libdl", "SHA", "URIParser"] +git-tree-sha1 = "12093ca6cdd0ee547c39b1870e0c9c3f154d9ca9" +uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee" +version = "0.8.10" + +[[BinaryProvider]] +deps = ["Libdl", "Logging", "SHA"] +git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648" +uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +version = "0.5.6" + +[[CSTParser]] +deps = ["Tokenize"] +git-tree-sha1 = "0ff80f68f55fcde2ed98d7b24d7abaf20727f3f8" +uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" +version = "0.6.1" + +[[CodecZlib]] +deps = ["BinaryProvider", "Libdl", "Test", "TranscodingStreams"] +git-tree-sha1 = "36bbf5374c661054d41410dc53ff752972583b9b" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.5.2" + +[[ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "10050a24b09e8e41b951e9976b109871ce98d965" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.8.0" + +[[Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Printf", "Reexport", "Test"] +git-tree-sha1 = "9f0a0210450acb91c730b730a994f8eef1d3d543" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.9.5" + +[[CommonSubexpressions]] +deps = ["Test"] +git-tree-sha1 = "efdaf19ab11c7889334ca247ff4c9f7c322817b0" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.2.0" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "2.1.0" + +[[Conda]] +deps = ["JSON", "VersionParsing"] +git-tree-sha1 = "9a11d428dcdc425072af4aea19ab1e8c3e01c032" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.3.0" + +[[Crayons]] +deps = ["Test"] +git-tree-sha1 = "f621b8ef51fd2004c7cf157ea47f027fdeac5523" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.0.0" + +[[DataAPI]] +git-tree-sha1 = "891a09f4f90361a28d0391c104a65c0202e22624" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.0.0" + +[[DataStructures]] +deps = ["InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "0809951a1774dc724da22d26e4289bbaab77809a" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.17.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[DiffResults]] +deps = ["Compat", "StaticArrays"] +git-tree-sha1 = "34a4a1e8be7bc99bc9c611b895b5baf37a80584c" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "0.0.4" + +[[DiffRules]] +deps = ["Random", "Test"] +git-tree-sha1 = "dc0869fb2f5b23466b32ea799bd82c76480167f7" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "0.0.10" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[FFTW]] +deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] +git-tree-sha1 = "e1a479d3c972f20c9a70563eec740bbfc786f515" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "0.3.0" + +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Test"] +git-tree-sha1 = "9ab8f76758cbabba8d7f103c51dce7f73fcf8e92" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.6.3" + +[[FixedPointNumbers]] +git-tree-sha1 = "d14a6fa5890ea3a7e5dcab6811114f132fec2b4b" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.6.1" + +[[Flux]] +deps = ["AbstractTrees", "Adapt", "CodecZlib", "Colors", "DelimitedFiles", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "Requires", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"] +git-tree-sha1 = "01debcc73af2428239343f5c3c0f8562529e91a0" +repo-rev = "zygote" +repo-url = "https://github.com/FluxML/Flux.jl.git" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.8.3" + +[[ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "InteractiveUtils", "LinearAlgebra", "NaNMath", "Random", "SparseArrays", "SpecialFunctions", "StaticArrays", "Test"] +git-tree-sha1 = "4c4d727f1b7e0092134fabfab6396b8945c1ea5b" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.3" + +[[IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "fcd9ef0f6595ceebb6b52e248440128cda354bf2" +repo-rev = "master" +repo-url = "https://github.com/MikeInnes/IRTools.jl.git" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.2.2" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.0" + +[[Juno]] +deps = ["Base64", "Logging", "Media", "Profile", "Test"] +git-tree-sha1 = "4e4a8d43aa7ecec66cadaf311fbd1e5c9d7b9175" +uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d" +version = "0.7.0" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[MacroTools]] +deps = ["CSTParser", "Compat", "DataStructures", "Test", "Tokenize"] +git-tree-sha1 = "d6e9dedb8c92c3465575442da456aec15a89ff76" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.1" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Media]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58" +uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27" +version = "0.5.0" + +[[Missings]] +deps = ["SparseArrays", "Test"] +git-tree-sha1 = "f0719736664b4358aa9ec173077d4285775f8007" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "0.4.1" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[NNlib]] +deps = ["Libdl", "LinearAlgebra", "Requires", "Statistics", "TimerOutputs"] +git-tree-sha1 = "0c667371391fc6bb31f7f12f96a56a17098b3de8" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.6.0" + +[[NaNMath]] +deps = ["Compat"] +git-tree-sha1 = "ce3b85e484a5d4c71dd5316215069311135fa9f2" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.2" + +[[OrderedCollections]] +deps = ["Random", "Serialization", "Test"] +git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.1.0" + +[[Parsers]] +deps = ["Dates", "Test"] +git-tree-sha1 = "db2b35dedab3c0e46dc15996d170af07a5ab91c9" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "0.3.6" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[Profile]] +deps = ["Printf"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Reexport]] +deps = ["Pkg"] +git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "0.2.0" + +[[Requires]] +deps = ["Test"] +git-tree-sha1 = "f6fbf4ba64d295e146e49e021207993b6b48c7d1" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "0.5.2" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SortingAlgorithms]] +deps = ["DataStructures", "Random", "Test"] +git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "0.3.1" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[SpecialFunctions]] +deps = ["BinDeps", "BinaryProvider", "Libdl", "Test"] +git-tree-sha1 = "0b45dc2e45ed77f445617b99ff2adf0f5b0f23ea" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "0.7.2" + +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "db23bbf50064c582b6f2b9b043c8e7e98ea8c0c6" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "0.11.0" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] +git-tree-sha1 = "c53e809e63fe5cf5de13632090bc3520649c9950" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.32.0" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[TimerOutputs]] +deps = ["Crayons", "Printf", "Test", "Unicode"] +git-tree-sha1 = "b80671c06f8f8bae08c55d67b5ce292c5ae2660c" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.0" + +[[Tokenize]] +git-tree-sha1 = "c8a8b00ae44a94950814ff77850470711a360225" +uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" +version = "0.5.5" + +[[TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.5" + +[[URIParser]] +deps = ["Test", "Unicode"] +git-tree-sha1 = "6ddf8244220dfda2f17539fa8c9de20d6c575b69" +uuid = "30578b45-9adc-5946-b283-645ec420af67" +version = "0.4.0" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[VersionParsing]] +deps = ["Compat"] +git-tree-sha1 = "c9d5aa108588b978bd859554660c8a5c4f2f7669" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.1.3" + +[[ZipFile]] +deps = ["BinaryProvider", "Libdl", "Printf"] +git-tree-sha1 = "580ce62b6c14244916cc28ad54f8a2e2886f843d" +uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" +version = "0.8.3" + +[[Zygote]] +deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "58d72525e3b3bf015c91f2b89dc5807291924c2f" +repo-rev = "master" +repo-url = "https://github.com/FluxML/Zygote.jl.git" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.3.2" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "24edeee7ac0e5537ce87054b0a6eda1c30cabfac" +repo-rev = "master" +repo-url = "https://github.com/FluxML/ZygoteRules.jl" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.1.0" diff --git a/Project.toml b/Project.toml index 73c9118..91f67ad 100644 --- a/Project.toml +++ b/Project.toml @@ -5,3 +5,15 @@ version = "0.1.0" [deps] Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +IRTools = "7869d1d1-7146-5819-86e3-90919afe41df" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" + +[compat] +julia = "≥ 1.0.0" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/src/DifferentiableNeuralComputer.jl b/src/DifferentiableNeuralComputer.jl new file mode 100644 index 0000000..7560da3 --- /dev/null +++ b/src/DifferentiableNeuralComputer.jl @@ -0,0 +1,12 @@ +module DifferentiableNeuralComputer + +import Flux +import Zygote + +struct DNC end + +function (dnc::DNC)(x) + return x +end + +end # module diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..f014d5d --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,5 @@ +using DifferentiableNeuralComputer +import Flux, Zygote +using Test + +@test_broken 1==2 From 37a78ac091a230859f00435dc1323b70eb58e5e6 Mon Sep 17 00:00:00 2001 From: Jules Merckx Date: Mon, 5 Aug 2019 15:21:32 +0200 Subject: [PATCH 2/4] implement lstm as in paper --- src/DifferentiableNeuralComputer.jl | 2 ++ src/lstm.jl | 40 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 src/lstm.jl diff --git a/src/DifferentiableNeuralComputer.jl b/src/DifferentiableNeuralComputer.jl index 7560da3..b3f0519 100644 --- a/src/DifferentiableNeuralComputer.jl +++ b/src/DifferentiableNeuralComputer.jl @@ -3,6 +3,8 @@ module DifferentiableNeuralComputer import Flux import Zygote +include("src/lstm.jl") + struct DNC end function (dnc::DNC)(x) diff --git a/src/lstm.jl b/src/lstm.jl new file mode 100644 index 0000000..3fd56c5 --- /dev/null +++ b/src/lstm.jl @@ -0,0 +1,40 @@ +using Flux +using Flux: glorot_uniform, @treelike, Recur + +mutable struct DNCLstmCell + Wi; Wf; Ws; Wo + bi; bf; bs; bo + h; s # These aren't used in the forward pass but need to be included for Recur. +end +@treelike DNCLstmCell + +function DNCLstmCell(in::Integer, hidden::Integer; init=glorot_uniform) + DNCLstmCell([init(hidden, in+2*hidden) for i in 1:4]..., + [zeros(hidden) for i in 1:6]...) +end + +#= + h: hidden at previous timestep + s: state at previous timestep + h′: state at previous layer (at same timestep) + x: original input to the LSTM +=# +function(c::DNCLstmCell)((h, s), (h′, x)) + i = σ.(c.Wi * [x; h; h′] .+ c.bi) + f = σ.(c.Wf * [x; h; h′] .+ c.bf) + s′ = f .* c.s .+ i .* tanh.(c.Ws * [x; h; h′] .+ c.bs) + o = σ.(c.Wo * [x; h; h′] .+ c.bo) + h′ = o .* tanh.(s′) + return((h′, s′), (h′, x)) +end + +Flux.hidden(m::DNCLstmCell) = (m.h, m.s) +DNCLstm(a...; ka...) = Recur(DNCLstmCell(a...; ka...)) + +#= +Note: for creating a multi-layered DNCLSTM, the arguments of each layer + need to be compatible. In other words, the hidden state has to keep + the same dimensions and the input size stays constant since x gets + passed through each layer. + e.g.: Chain(DNCLSTM(16,10), DNCLSTM(16,10)) +=# From 7d95be345092623121831fed222528ce58bd45e6 Mon Sep 17 00:00:00 2001 From: Jules Merckx Date: Mon, 5 Aug 2019 18:28:25 +0200 Subject: [PATCH 3/4] lstm: batching and testing --- src/DifferentiableNeuralComputer.jl | 2 +- src/lstm.jl | 4 +++- test/runtests.jl | 9 +++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/DifferentiableNeuralComputer.jl b/src/DifferentiableNeuralComputer.jl index b3f0519..78e2ea2 100644 --- a/src/DifferentiableNeuralComputer.jl +++ b/src/DifferentiableNeuralComputer.jl @@ -3,7 +3,7 @@ module DifferentiableNeuralComputer import Flux import Zygote -include("src/lstm.jl") +include("lstm.jl") struct DNC end diff --git a/src/lstm.jl b/src/lstm.jl index 3fd56c5..a8b6aac 100644 --- a/src/lstm.jl +++ b/src/lstm.jl @@ -19,7 +19,9 @@ end h′: state at previous layer (at same timestep) x: original input to the LSTM =# -function(c::DNCLstmCell)((h, s), (h′, x)) +function(c::DNCLstmCell)((h, s), h′, x) + h = size(h)==size(h′) ? h : repeat(h, 1, size(x, 2)) # support for batching. + i = σ.(c.Wi * [x; h; h′] .+ c.bi) f = σ.(c.Wf * [x; h; h′] .+ c.bf) s′ = f .* c.s .+ i .* tanh.(c.Ws * [x; h; h′] .+ c.bs) diff --git a/test/runtests.jl b/test/runtests.jl index f014d5d..074dc2b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,4 +2,13 @@ using DifferentiableNeuralComputer import Flux, Zygote using Test +using DifferentiableNeuralComputer: DNCLstm + @test_broken 1==2 + +begin + m = DNCLstm(3, 5) + @test m.state == m.init + @test size.(m(rand(5, 4), rand(3, 4))) == ((5, 4), (3,4)) + @test m.state != m.init +end From be0c1fbf9fb6751180956f48326515cc7f69067b Mon Sep 17 00:00:00 2001 From: Jules Merckx Date: Tue, 6 Aug 2019 21:42:01 +0200 Subject: [PATCH 4/4] LSTM: namechanges, changing tests --- src/{lstm.jl => DNCLSTM.jl} | 16 +++++++++------- src/DifferentiableNeuralComputer.jl | 2 +- test/runtests.jl | 12 ++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) rename src/{lstm.jl => DNCLSTM.jl} (68%) diff --git a/src/lstm.jl b/src/DNCLSTM.jl similarity index 68% rename from src/lstm.jl rename to src/DNCLSTM.jl index a8b6aac..0db42d1 100644 --- a/src/lstm.jl +++ b/src/DNCLSTM.jl @@ -1,15 +1,15 @@ using Flux using Flux: glorot_uniform, @treelike, Recur -mutable struct DNCLstmCell +mutable struct DNCLSTMCell Wi; Wf; Ws; Wo bi; bf; bs; bo h; s # These aren't used in the forward pass but need to be included for Recur. end -@treelike DNCLstmCell +@treelike DNCLSTMCell -function DNCLstmCell(in::Integer, hidden::Integer; init=glorot_uniform) - DNCLstmCell([init(hidden, in+2*hidden) for i in 1:4]..., +function DNCLSTMCell(in::Integer, hidden::Integer; init=glorot_uniform) + DNCLSTMCell([init(hidden, in+2*hidden) for i in 1:4]..., [zeros(hidden) for i in 1:6]...) end @@ -19,7 +19,7 @@ end h′: state at previous layer (at same timestep) x: original input to the LSTM =# -function(c::DNCLstmCell)((h, s), h′, x) +function (c::DNCLSTMCell)((h, s), (h′, x)) h = size(h)==size(h′) ? h : repeat(h, 1, size(x, 2)) # support for batching. i = σ.(c.Wi * [x; h; h′] .+ c.bi) @@ -30,9 +30,10 @@ function(c::DNCLstmCell)((h, s), h′, x) return((h′, s′), (h′, x)) end -Flux.hidden(m::DNCLstmCell) = (m.h, m.s) -DNCLstm(a...; ka...) = Recur(DNCLstmCell(a...; ka...)) +# Note: input needs to be a tuple containing hidden state and original input. +Flux.hidden(m::DNCLSTMCell) = (m.h, m.s) +DNCLSTM(a...; ka...) = Recur(DNCLSTMCell(a...; ka...)) #= Note: for creating a multi-layered DNCLSTM, the arguments of each layer need to be compatible. In other words, the hidden state has to keep @@ -40,3 +41,4 @@ Note: for creating a multi-layered DNCLSTM, the arguments of each layer passed through each layer. e.g.: Chain(DNCLSTM(16,10), DNCLSTM(16,10)) =# +DNCLSTM(in, hidden, nlayers; ka...) = Chain([DNCLSTM(in, hidden; ka...) for i in 1:nlayers]...) diff --git a/src/DifferentiableNeuralComputer.jl b/src/DifferentiableNeuralComputer.jl index 78e2ea2..246daa7 100644 --- a/src/DifferentiableNeuralComputer.jl +++ b/src/DifferentiableNeuralComputer.jl @@ -3,7 +3,7 @@ module DifferentiableNeuralComputer import Flux import Zygote -include("lstm.jl") +include("DNCLSTM.jl") struct DNC end diff --git a/test/runtests.jl b/test/runtests.jl index 074dc2b..9e3170f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,13 +2,13 @@ using DifferentiableNeuralComputer import Flux, Zygote using Test -using DifferentiableNeuralComputer: DNCLstm +using DifferentiableNeuralComputer: DNCLSTM @test_broken 1==2 -begin - m = DNCLstm(3, 5) - @test m.state == m.init - @test size.(m(rand(5, 4), rand(3, 4))) == ((5, 4), (3,4)) - @test m.state != m.init +@testset "DNCLSTM" begin + m = DNCLSTM(3, 5, 2) + @test m[2].state == m[2].init + @test size.(m((rand(5, 4), rand(3, 4)))) == ((5, 4), (3,4)) + @test m[2].state != m[2].init end